diff --git a/pkg/codecs/h264/dts_extractor.go b/pkg/codecs/h264/dts_extractor.go index 657f99d..d50f516 100644 --- a/pkg/codecs/h264/dts_extractor.go +++ b/pkg/codecs/h264/dts_extractor.go @@ -4,77 +4,11 @@ import ( "bytes" "fmt" "time" - - "github.com/bluenviron/mediacommon/pkg/bits" -) - -const ( - maxReorderedFrames = 10 - /* - (max_size(first_mb_in_slice) + max_size(slice_type) + max_size(pic_parameter_set_id) + - max_size(frame_num) + max_size(pic_order_cnt_lsb)) * 4 / 3 = - (3 * max_size(golomb) + (max(Log2MaxFrameNumMinus4) + 4) / 8 + (max(Log2MaxPicOrderCntLsbMinus4) + 4) / 8) * 4 / 3 = - (3 * 4 + 2 + 2) * 4 / 3 = 22 - */ - maxBytesToGetPOC = 22 ) -func getPictureOrderCount(buf []byte, sps *SPS, idr bool) (uint32, error) { - buf = buf[1:] - lb := len(buf) - - if lb > maxBytesToGetPOC { - lb = maxBytesToGetPOC - } - - buf = EmulationPreventionRemove(buf[:lb]) - pos := 0 - - _, err := bits.ReadGolombUnsigned(buf, &pos) // first_mb_in_slice - if err != nil { - return 0, err - } - - _, err = bits.ReadGolombUnsigned(buf, &pos) // slice_type - if err != nil { - return 0, err - } - - _, err = bits.ReadGolombUnsigned(buf, &pos) // pic_parameter_set_id - if err != nil { - return 0, err - } - - _, err = bits.ReadBits(buf, &pos, int(sps.Log2MaxFrameNumMinus4+4)) // frame_num - if err != nil { - return 0, err - } - - if idr { - _, err = bits.ReadGolombUnsigned(buf, &pos) // idr_pic_id - if err != nil { - return 0, err - } - } - - picOrderCntLsb, err := bits.ReadBits(buf, &pos, int(sps.Log2MaxPicOrderCntLsbMinus4+4)) - if err != nil { - return 0, err - } - - return uint32(picOrderCntLsb), nil -} - -func getPictureOrderCountDiff(a uint32, b uint32, sps *SPS) int32 { - max := uint32(1 << (sps.Log2MaxPicOrderCntLsbMinus4 + 4)) - d := (a - b) & (max - 1) - if d > (max / 2) { - return int32(d) - int32(max) - } - return int32(d) -} - // DTSExtractor allows to extract DTS from PTS. +// +// Deprecated: replaced by DTSExtractor2. type DTSExtractor struct { sps []byte spsp *SPS @@ -87,6 +21,8 @@ type DTSExtractor struct { } // NewDTSExtractor allocates a DTSExtractor. +// +// Deprecated: replaced by NewDTSExtractor2. func NewDTSExtractor() *DTSExtractor { return &DTSExtractor{ pocIncrement: 2, diff --git a/pkg/codecs/h264/dts_extractor2.go b/pkg/codecs/h264/dts_extractor2.go new file mode 100644 index 0000000..ebfab63 --- /dev/null +++ b/pkg/codecs/h264/dts_extractor2.go @@ -0,0 +1,241 @@ +package h264 + +import ( + "bytes" + "fmt" + + "github.com/bluenviron/mediacommon/pkg/bits" +) + +const ( + maxReorderedFrames = 10 + /* + (max_size(first_mb_in_slice) + max_size(slice_type) + max_size(pic_parameter_set_id) + + max_size(frame_num) + max_size(pic_order_cnt_lsb)) * 4 / 3 = + (3 * max_size(golomb) + (max(Log2MaxFrameNumMinus4) + 4) / 8 + (max(Log2MaxPicOrderCntLsbMinus4) + 4) / 8) * 4 / 3 = + (3 * 4 + 2 + 2) * 4 / 3 = 22 + */ + maxBytesToGetPOC = 22 +) + +func getPictureOrderCount(buf []byte, sps *SPS, idr bool) (uint32, error) { + buf = buf[1:] + lb := len(buf) + + if lb > maxBytesToGetPOC { + lb = maxBytesToGetPOC + } + + buf = EmulationPreventionRemove(buf[:lb]) + pos := 0 + + _, err := bits.ReadGolombUnsigned(buf, &pos) // first_mb_in_slice + if err != nil { + return 0, err + } + + _, err = bits.ReadGolombUnsigned(buf, &pos) // slice_type + if err != nil { + return 0, err + } + + _, err = bits.ReadGolombUnsigned(buf, &pos) // pic_parameter_set_id + if err != nil { + return 0, err + } + + _, err = bits.ReadBits(buf, &pos, int(sps.Log2MaxFrameNumMinus4+4)) // frame_num + if err != nil { + return 0, err + } + + if idr { + _, err = bits.ReadGolombUnsigned(buf, &pos) // idr_pic_id + if err != nil { + return 0, err + } + } + + picOrderCntLsb, err := bits.ReadBits(buf, &pos, int(sps.Log2MaxPicOrderCntLsbMinus4+4)) + if err != nil { + return 0, err + } + + return uint32(picOrderCntLsb), nil +} + +func getPictureOrderCountDiff(a uint32, b uint32, sps *SPS) int32 { + max := uint32(1 << (sps.Log2MaxPicOrderCntLsbMinus4 + 4)) + d := (a - b) & (max - 1) + if d > (max / 2) { + return int32(d) - int32(max) + } + return int32(d) +} + +// DTSExtractor2 computes DTS from PTS. +type DTSExtractor2 struct { + sps []byte + spsp *SPS + prevDTSFilled bool + prevDTS int64 + expectedPOC uint32 + reorderedFrames int + pauseDTS int + pocIncrement int +} + +// NewDTSExtractor2 allocates a DTSExtractor. +func NewDTSExtractor2() *DTSExtractor2 { + return &DTSExtractor2{ + pocIncrement: 2, + } +} + +func (d *DTSExtractor2) extractInner(au [][]byte, pts int64) (int64, bool, error) { + var idr []byte + var nonIDR []byte + // a value of 00 indicates that the content of the NAL unit is not + // used to reconstruct reference pictures for inter picture + // prediction. Such NAL units can be discarded without risking + // the integrity of the reference pictures. Values greater than + // 00 indicate that the decoding of the NAL unit is required to + // maintain the integrity of the reference pictures. + nonZeroNalRefIDFound := false + + for _, nalu := range au { + typ := NALUType(nalu[0] & 0x1F) + nonZeroNalRefIDFound = nonZeroNalRefIDFound || ((nalu[0] & 0x60) > 0) + switch typ { + case NALUTypeSPS: + if !bytes.Equal(d.sps, nalu) { + var spsp SPS + err := spsp.Unmarshal(nalu) + if err != nil { + return 0, false, fmt.Errorf("invalid SPS: %w", err) + } + d.sps = nalu + d.spsp = &spsp + + // reset state + d.reorderedFrames = 0 + d.pocIncrement = 2 + } + + case NALUTypeIDR: + idr = nalu + + case NALUTypeNonIDR: + nonIDR = nalu + } + } + + if d.spsp == nil { + return 0, false, fmt.Errorf("SPS not received yet") + } + + if d.spsp.PicOrderCntType == 2 || !d.spsp.FrameMbsOnlyFlag { + return pts, false, nil + } + + if d.spsp.PicOrderCntType == 1 { + return 0, false, fmt.Errorf("pic_order_cnt_type = 1 is not supported yet") + } + + // Implicit processing of PicOrderCountType 0 + switch { + case idr != nil: + d.pauseDTS = 0 + + var err error + d.expectedPOC, err = getPictureOrderCount(idr, d.spsp, true) + if err != nil { + return 0, false, err + } + + if !d.prevDTSFilled || d.reorderedFrames == 0 { + return pts, false, nil + } + + return d.prevDTS + (pts-d.prevDTS)/int64(d.reorderedFrames+1), false, nil + + case nonIDR != nil: + d.expectedPOC += uint32(d.pocIncrement) + d.expectedPOC &= ((1 << (d.spsp.Log2MaxPicOrderCntLsbMinus4 + 4)) - 1) + + if d.pauseDTS > 0 { + d.pauseDTS-- + return d.prevDTS + 90, true, nil + } + + poc, err := getPictureOrderCount(nonIDR, d.spsp, false) + if err != nil { + return 0, false, err + } + + if d.pocIncrement == 2 && (poc%2) != 0 { + d.pocIncrement = 1 + d.expectedPOC /= 2 + } + + pocDiff := int(getPictureOrderCountDiff(poc, d.expectedPOC, d.spsp)) / d.pocIncrement + limit := -(d.reorderedFrames + 1) + + // this happens when there are B-frames immediately following an IDR frame + if pocDiff < limit { + increase := limit - pocDiff + if (d.reorderedFrames + increase) > maxReorderedFrames { + return 0, false, fmt.Errorf("too many reordered frames (%d)", d.reorderedFrames+increase) + } + + d.reorderedFrames += increase + d.pauseDTS = increase + return d.prevDTS + 90, true, nil + } + + if pocDiff == limit { + return pts, false, nil + } + + if pocDiff > d.reorderedFrames { + increase := pocDiff - d.reorderedFrames + if (d.reorderedFrames + increase) > maxReorderedFrames { + return 0, false, fmt.Errorf("too many reordered frames (%d)", d.reorderedFrames+increase) + } + + d.reorderedFrames += increase + d.pauseDTS = increase - 1 + return d.prevDTS + 90, false, nil + } + + return d.prevDTS + (pts-d.prevDTS)/int64(pocDiff+d.reorderedFrames+1), false, nil + + case !nonZeroNalRefIDFound: + return d.prevDTS, false, nil + + default: + return 0, false, fmt.Errorf("access unit doesn't contain an IDR or non-IDR NALU") + } +} + +// Extract extracts the DTS of an access unit. +func (d *DTSExtractor2) Extract(au [][]byte, pts int64) (int64, error) { + dts, skipChecks, err := d.extractInner(au, pts) + if err != nil { + return 0, err + } + + if !skipChecks && dts > pts { + return 0, fmt.Errorf("DTS is greater than PTS") + } + + if d.prevDTSFilled && dts < d.prevDTS { + return 0, fmt.Errorf("DTS is not monotonically increasing, was %v, now is %v", + d.prevDTS, dts) + } + + d.prevDTS = dts + d.prevDTSFilled = true + + return dts, err +} diff --git a/pkg/codecs/h264/dts_extractor2_test.go b/pkg/codecs/h264/dts_extractor2_test.go new file mode 100644 index 0000000..9514d19 --- /dev/null +++ b/pkg/codecs/h264/dts_extractor2_test.go @@ -0,0 +1,472 @@ +package h264 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type sample2 struct { + au [][]byte + dts int64 + pts int64 +} + +var casesDTSExtractor2 = []struct { + name string + sequence []sample2 +}{ + { + "with timing info", + []sample2{ + { + [][]byte{ + { // SPS + 0x67, 0x64, 0x00, 0x28, 0xac, 0xd9, 0x40, 0x78, + 0x02, 0x27, 0xe5, 0x84, 0x00, 0x00, 0x03, 0x00, + 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, 0x60, + 0xc6, 0x58, + }, + { // IDR + 0x65, 0x88, 0x84, 0x00, 0x33, 0xff, + }, + }, + int64(333333333 * time.Nanosecond * 90000 / time.Second), + int64(333333333 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x41, 0x9a, 0x21, 0x6c, 0x45, 0xff}}, + int64(366666666 * time.Nanosecond * 90000 / time.Second), + int64(366666666 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x41, 0x9a, 0x42, 0x3c, 0x21, 0x93}}, + int64(400000000 * time.Nanosecond * 90000 / time.Second), + int64(400000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x41, 0x9a, 0x63, 0x49, 0xe1, 0x0f}}, + int64(433333333 * time.Nanosecond * 90000 / time.Second), + int64(433333333 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x41, 0x9a, 0x86, 0x49, 0xe1, 0x0f}}, + int64(434333333 * time.Nanosecond * 90000 / time.Second), + int64(533333333 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x41, 0x9e, 0xa5, 0x42, 0x7f, 0xf9}}, + int64(435333333 * time.Nanosecond * 90000 / time.Second), + int64(500000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x01, 0x9e, 0xc4, 0x69, 0x13, 0xff}}, + int64(466666666 * time.Nanosecond * 90000 / time.Second), + int64(466666666 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x41, 0x9a, 0xc8, 0x4b, 0xa8, 0x42}}, + int64(499999999 * time.Nanosecond * 90000 / time.Second), + int64(600000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{ + { // IDR + 0x65, 0x88, 0x84, 0x00, 0x33, 0xff, + }, + }, + int64(533333332 * time.Nanosecond * 90000 / time.Second), + int64(599999999 * time.Nanosecond * 90000 / time.Second), + }, + }, + }, + { + "no timing info", + []sample2{ + { + [][]byte{ + { // SPS + 0x27, 0x64, 0x00, 0x20, 0xac, 0x52, 0x18, 0x0f, + 0x01, 0x17, 0xef, 0xff, 0x00, 0x01, 0x00, 0x01, + 0x6a, 0x02, 0x02, 0x03, 0x6d, 0x85, 0x6b, 0xde, + 0xf8, 0x08, + }, + { // IDR + 0x25, 0xb8, 0x08, 0x02, 0x1f, 0xff, + }, + }, + int64(850000000 * time.Nanosecond * 90000 / time.Second), + int64(850000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe1, 0x05, 0xc7, 0x38, 0xbf}}, + int64(866666667 * time.Nanosecond * 90000 / time.Second), + int64(866666667 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe2, 0x09, 0xa1, 0xce, 0x0b}}, + int64(883333334 * time.Nanosecond * 90000 / time.Second), + int64(883333334 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe3, 0x0d, 0xb1, 0xce, 0x02}}, + int64(900000000 * time.Nanosecond * 90000 / time.Second), + int64(900000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe4, 0x11, 0x90, 0x73, 0x80}}, + int64(916666667 * time.Nanosecond * 90000 / time.Second), + int64(916666667 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe5, 0x19, 0x0e, 0x70, 0x01}}, + int64(917666667 * time.Nanosecond * 90000 / time.Second), + int64(950000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x01, 0xa9, 0x85, 0x7c, 0x93, 0xff}}, + int64(933333334 * time.Nanosecond * 90000 / time.Second), + int64(933333334 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe6, 0x1d, 0x0e, 0x70, 0x01}}, + int64(950000000 * time.Nanosecond * 90000 / time.Second), + int64(966666667 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe7, 0x21, 0x0e, 0x70, 0x01}}, + int64(966666667 * time.Nanosecond * 90000 / time.Second), + int64(983333334 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe8, 0x25, 0x0e, 0x70, 0x01}}, + int64(983333333 * time.Nanosecond * 90000 / time.Second), + int64(999999999 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xe9, 0x29, 0x0e, 0x70, 0x01}}, + int64(999999999 * time.Nanosecond * 90000 / time.Second), + int64(1016666667 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x21, 0xea, 0x31, 0x0e, 0x70, 0x01}}, + int64(1016666666 * time.Nanosecond * 90000 / time.Second), + int64(1050000000 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{0x01, 0xaa, 0xcb, 0x7c, 0x93, 0xff}}, + int64(1033333334 * time.Nanosecond * 90000 / time.Second), + int64(1033333334 * time.Nanosecond * 90000 / time.Second), + }, + }, + }, + { + "poc increment = 1", + []sample2{ + { + [][]byte{ + { // SPS + 0x67, 0x64, 0x00, 0x2a, 0xac, 0x2c, 0x6a, 0x81, + 0xe0, 0x08, 0x9f, 0x96, 0x6e, 0x02, 0x02, 0x02, + 0x80, 0x00, 0x03, 0x84, 0x00, 0x00, 0xaf, 0xc8, + 0x02, + }, + { // IDR + 0x65, 0xb8, 0x00, 0x00, 0x0b, 0xc8, 0x00, 0x00, 0x00, + }, + }, + int64(61 * time.Millisecond * 90000 / time.Second), + int64(61 * time.Millisecond * 90000 / time.Second), + }, + { + [][]byte{{0x61, 0xe0, 0x20, 0x00, 0x39, 0x37}}, + int64(101 * time.Millisecond * 90000 / time.Second), + int64(101 * time.Millisecond * 90000 / time.Second), + }, + { + [][]byte{{0x61, 0xe0, 0x40, 0x00, 0x59, 0x37}}, + int64(141 * time.Millisecond * 90000 / time.Second), + int64(141 * time.Millisecond * 90000 / time.Second), + }, + { + [][]byte{{0x61, 0xe0, 0x60, 0x00, 0x79, 0x37}}, + int64(181 * time.Millisecond * 90000 / time.Second), + int64(181 * time.Millisecond * 90000 / time.Second), + }, + }, + }, + { + "B-frames after IDR (OBS 29.1.3 QuickSync on Windows)", + []sample2{ + { + [][]byte{ + { // SPS + 0x27, 0x64, 0x00, 0x2a, 0xac, 0x2d, 0x90, 0x07, + 0x80, 0x22, 0x7e, 0x5c, 0x05, 0xa8, 0x08, 0x08, + 0x0a, 0x00, 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, + 0x03, 0x00, 0xf1, 0xd0, 0x80, 0x04, 0xc4, 0x80, + 0x00, 0x09, 0x89, 0x68, 0xde, 0xf7, 0xc1, 0xda, + 0x1c, 0x31, 0x92, + }, + { // IDR + 0x65, 0x88, 0x80, 0x14, 0x3, 0xff, 0xde, 0x8, 0xe4, 0x74, + }, + }, + int64(1916 * time.Millisecond * 90000 / time.Second), + int64(1916 * time.Millisecond * 90000 / time.Second), + }, + { // b-frame + [][]byte{{0x41, 0x9e, 0x3, 0xe4, 0x3f, 0x0, 0x0, 0x3, 0x0, 0x0}}, + int64(1917 * time.Millisecond * 90000 / time.Second), + int64(1883 * time.Millisecond * 90000 / time.Second), + }, + { // b-frame + [][]byte{{0x1, 0x9e, 0x5, 0xd4, 0x7f, 0x0, 0x0, 0x3, 0x0, 0x0}}, + int64(1918 * time.Millisecond * 90000 / time.Second), + int64(1867 * time.Millisecond * 90000 / time.Second), + }, + { // p-frame + [][]byte{{0x1, 0x9e, 0x5, 0xf4, 0x7f, 0x0, 0x0, 0x3, 0x0, 0x0}}, + int64(1919 * time.Millisecond * 90000 / time.Second), + int64(1899 * time.Millisecond * 90000 / time.Second), + }, + { // p-frame + [][]byte{{0x1, 0x9e, 0x5, 0xf4, 0x7f, 0x0, 0x0, 0x3, 0x0, 0x0}}, + int64(1920 * time.Millisecond * 90000 / time.Second), + int64(1983 * time.Millisecond * 90000 / time.Second), + }, + }, + }, + { + "mbs_only_flag = 0", + []sample2{ + { + [][]byte{ + { // SPS + 0x67, 0x4d, 0x40, 0x28, 0xab, 0x60, 0x3c, 0x02, + 0x23, 0xef, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, + 0x10, 0x00, 0x00, 0x03, 0x03, 0x2e, 0x94, 0x00, + 0x35, 0x64, 0x06, 0xb2, 0x85, 0x08, 0x0e, 0xe2, + 0xc5, 0x22, 0xc0, + }, + {0x68, 0xca, 0x41, 0xf2}, // PPS + {0x6, 0x0, 0x6, 0x85, 0x7e, 0x40, 0x0, 0x0, 0x10, 0x1}, // SEI + {0x65, 0x88, 0x82, 0x80, 0x1f, 0xff, 0xfb, 0xf0, 0xa2, 0x88}, // IDR + {0x6, 0x1, 0x2, 0x4, 0x24, 0x80}, // SEI + {0x41, 0x9a, 0xc, 0x1c, 0x2f, 0xe4, 0xed, 0x23, 0xb5, 0x63}, // non-IDR + }, + 0, + 0, + }, + { + [][]byte{ + {0x6, 0x1, 0x2, 0x8, 0x14, 0x80}, // SEI + {0x41, 0x9a, 0x18, 0x2a, 0x1f, 0xeb, 0x2f, 0xa2, 0xb1, 0x7e}, // non-IDR + }, + int64(40 * time.Millisecond * 90000 / time.Second), + int64(40 * time.Millisecond * 90000 / time.Second), + }, + { + [][]byte{ + {0x6, 0x1, 0x2, 0xc, 0x24, 0x80}, // SEI + {0x41, 0x9a, 0x1c, 0x3a, 0xf, 0xfa, 0x55, 0xc2, 0x55, 0xea}, // non-IDR + }, + int64(80 * time.Millisecond * 90000 / time.Second), + int64(80 * time.Millisecond * 90000 / time.Second), + }, + }, + }, + { + "Log2MaxPicOrderCntLsbMinus4 = 12", + []sample2{ + { + [][]byte{ + { // SPS + 0x67, 0x42, 0xc0, 0x1e, 0x8c, 0x8d, 0x40, 0x50, 0x17, 0xfc, 0xb0, 0x0f, 0x08, 0x84, 0x6a, + }, + { // PPS + 0x68, 0xce, 0x3c, 0x80, + }, + { // IDR + 0x65, 0x88, 0x80, 0x14, 0x3, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + }, + 0, + 0, + }, + { + [][]byte{ + {0x61, 0x00, 0xf0, 0xe0, 0x00, 0x40, 0x00, 0xbe, 0x47, 0x9b}, // non-IDR + }, + int64(40 * time.Millisecond * 90000 / time.Second), + int64(40 * time.Millisecond * 90000 / time.Second), + }, + }, + }, + { + "issue mediamtx/3094 (non-zero IDR POC)", + []sample2{ + { + [][]byte{ + { + 0x67, 0x42, 0x80, 0x28, 0x8c, 0x8d, 0x40, 0x5a, + 0x09, 0x22, + }, + { + 0x68, 0xce, 0x3c, 0x80, + }, + { + 0x65, 0xb8, 0x00, 0x0c, 0xa2, 0x40, 0x33, 0x93, + 0x14, 0x00, 0x04, 0x1a, 0x6d, 0x6d, 0x6d, 0x6d, + 0x6d, 0x6d, 0x5d, 0xaa, 0xb5, 0xb5, 0xb5, 0xb5, + 0xb5, 0xb5, 0xb5, 0xb5, 0xb5, 0xb5, 0xb5, 0xb5, + 0xb5, 0xb5, 0x76, 0xb6, 0xb6, 0xb6, 0xaa, 0xd6, + 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, + 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, 0xd6, 0xd5, + 0xda, 0xda, 0xaa, 0x7a, 0x7a, 0x7a, 0x7a, 0x7a, + 0x79, 0x1e, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, + 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, + 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, + 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, + 0xde, 0xde, 0xde, 0xde, + }, + }, + 0, + 0, + }, + { + [][]byte{ + { + 0x41, 0xe0, 0x00, 0x65, 0x12, 0x80, 0xce, 0x78, + 0x16, 0x00, 0x99, 0xff, 0xff, 0xff, 0xe0, 0xe4, + 0x1a, 0x7f, 0xff, 0xff, 0xea, 0x11, 0x01, 0x01, + 0xff, 0xff, 0xfc, 0x20, 0x08, 0x3f, 0xff, 0xff, + 0xfc, 0x0f, 0x22, 0x7f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xb8, 0x60, 0x04, 0x87, + 0x02, 0xc8, 0x18, 0x38, 0x60, 0x04, 0x87, 0x03, + 0x00, 0x35, 0xa8, 0x16, 0x40, 0x9b, 0x04, 0xd0, + 0x11, 0x00, 0x24, 0x38, 0x11, 0x01, 0x6c, 0x16, + 0x41, 0x60, 0x2c, 0x82, 0xd8, 0x2c, 0x05, 0x90, + 0x5b, 0x05, 0x80, 0xb2, 0x0b, 0x60, 0xb0, 0x16, + 0x41, 0x6c, 0x16, 0x02, 0xc8, 0x2d, 0x82, 0xc0, + 0x59, 0x05, 0xb0, 0x58, + }, + }, + int64(120 * time.Millisecond * 90000 / time.Second), + int64(120 * time.Millisecond * 90000 / time.Second), + }, + { + [][]byte{ + { + 0x41, 0xe0, 0x00, 0xa5, 0x13, 0x00, 0xce, 0xf0, + 0x2c, 0x70, 0x20, 0x01, 0x43, 0xc0, 0x8b, 0xc3, + 0x01, 0x99, 0x60, 0x80, 0x04, 0x07, 0x06, 0x39, + 0xe0, 0x80, 0x04, 0x04, 0x37, 0x80, 0x90, 0xe4, + 0x06, 0x9c, 0xa0, 0x23, 0x60, 0x06, 0x25, 0x80, + }, + }, + int64(160 * time.Millisecond * 90000 / time.Second), + int64(160 * time.Millisecond * 90000 / time.Second), + }, + }, + }, + { + "issue mediamtx/3614 Only SEI received", + []sample2{ + { + [][]byte{ + { // SPS + 0x27, 0x64, 0x00, 0x2a, 0xac, 0x2d, 0x90, 0x07, + 0x80, 0x22, 0x7e, 0x5c, 0x05, 0xa8, 0x08, 0x08, + 0x0a, 0x00, 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, + 0x03, 0x00, 0xf1, 0xd0, 0x80, 0x04, 0xc4, 0x80, + 0x00, 0x09, 0x89, 0x68, 0xde, 0xf7, 0xc1, 0xda, + 0x1c, 0x31, 0x92, + }, + {0x68, 0xca, 0x41, 0xf2}, // PPS + {0x6, 0x0, 0x6, 0x85, 0x7e, 0x40, 0x0, 0x0, 0x10, 0x1}, // SEI + {0x65, 0x88, 0x82, 0x80, 0x1f, 0xff, 0xfb, 0xf0, 0xa2, 0x88}, // IDR + {0x6, 0x1, 0x2, 0x4, 0x24, 0x80}, // SEI + {0x41, 0x9a, 0xc, 0x1c, 0x2f, 0xe4, 0xed, 0x23, 0xb5, 0x63}, // non-IDR + }, + 0, + 0, + }, + { + [][]byte{ + {0x6, 0x1, 0x2, 0x8, 0x14, 0x80}, // SEI + }, + 0, + 0, + }, + { + [][]byte{ + {0x61, 0x00, 0xf0, 0xe0, 0x00, 0x40, 0x00, 0xbe, 0x47, 0x9b}, // non-IDR + }, + int64(40 * time.Millisecond * 90000 / time.Second), + int64(40 * time.Millisecond * 90000 / time.Second), + }, + { + [][]byte{ + {0x6, 0x1, 0x2, 0x8, 0x14, 0x80}, // SEI + }, + int64(40 * time.Millisecond * 90000 / time.Second), + int64(80 * time.Millisecond * 90000 / time.Second), + }, + }, + }, +} + +func TestDTSExtractor2(t *testing.T) { + for _, ca := range casesDTSExtractor2 { + t.Run(ca.name, func(t *testing.T) { + ex := NewDTSExtractor2() + for _, sample := range ca.sequence { + dts, err := ex.Extract(sample.au, sample.pts) + require.NoError(t, err) + require.Equal(t, sample.dts, dts) + } + }) + } +} + +func FuzzDTSExtractor2FirstAU(f *testing.F) { + f.Fuzz(func(_ *testing.T, a []byte, b []byte) { + if len(a) < 1 || len(b) < 1 { + return + } + + ex := NewDTSExtractor2() + + ex.Extract([][]byte{ //nolint:errcheck + a, + b, + }, 0) + }) +} + +func FuzzDTSExtractor2SecondAU(f *testing.F) { + f.Fuzz(func(t *testing.T, a []byte) { + if len(a) < 1 { + return + } + + ex := NewDTSExtractor2() + + _, err := ex.Extract([][]byte{ + { // SPS + 0x27, 0x64, 0x00, 0x20, 0xac, 0x52, 0x18, 0x0f, + 0x01, 0x17, 0xef, 0xff, 0x00, 0x01, 0x00, 0x01, + 0x6a, 0x02, 0x02, 0x03, 0x6d, 0x85, 0x6b, 0xde, + 0xf8, 0x08, + }, + { // IDR + 0x25, 0xb8, 0x08, 0x02, 0x1f, 0xff, + }, + }, 0) + require.NoError(t, err) + + ex.Extract([][]byte{a}, int64(1*time.Second*90000/time.Second)) //nolint:errcheck + }) +} diff --git a/pkg/codecs/h265/dts_extractor.go b/pkg/codecs/h265/dts_extractor.go index 519228e..6cc1d80 100644 --- a/pkg/codecs/h265/dts_extractor.go +++ b/pkg/codecs/h265/dts_extractor.go @@ -2,138 +2,12 @@ package h265 import ( "fmt" - "math" "time" - - "github.com/bluenviron/mediacommon/pkg/bits" - "github.com/bluenviron/mediacommon/pkg/codecs/h264" -) - -const ( - maxBytesToGetPOC = 12 ) -func getPTSDTSDiff(buf []byte, sps *SPS, pps *PPS) (uint32, error) { - typ := NALUType((buf[0] >> 1) & 0b111111) - - buf = buf[1:] - lb := len(buf) - - if lb > maxBytesToGetPOC { - lb = maxBytesToGetPOC - } - - buf = h264.EmulationPreventionRemove(buf[:lb]) - pos := 8 - - firstSliceSegmentInPicFlag, err := bits.ReadFlag(buf, &pos) - if err != nil { - return 0, err - } - - if !firstSliceSegmentInPicFlag { - return 0, fmt.Errorf("first_slice_segment_in_pic_flag = 0 is not supported") - } - - if typ >= NALUType_BLA_W_LP && typ <= NALUType_RSV_IRAP_VCL23 { - _, err = bits.ReadFlag(buf, &pos) // no_output_of_prior_pics_flag - if err != nil { - return 0, err - } - } - - _, err = bits.ReadGolombUnsigned(buf, &pos) // slice_pic_parameter_set_id - if err != nil { - return 0, err - } - - if pps.NumExtraSliceHeaderBits > 0 { - err = bits.HasSpace(buf, pos, int(pps.NumExtraSliceHeaderBits)) - if err != nil { - return 0, err - } - pos += int(pps.NumExtraSliceHeaderBits) - } - - sliceType, err := bits.ReadGolombUnsigned(buf, &pos) // slice_type - if err != nil { - return 0, err - } - - if pps.OutputFlagPresentFlag { - _, err = bits.ReadFlag(buf, &pos) // pic_output_flag - if err != nil { - return 0, err - } - } - - if sps.SeparateColourPlaneFlag { - _, err = bits.ReadBits(buf, &pos, 2) // colour_plane_id - if err != nil { - return 0, err - } - } - - _, err = bits.ReadBits(buf, &pos, int(sps.Log2MaxPicOrderCntLsbMinus4+4)) // pic_order_cnt_lsb - if err != nil { - return 0, err - } - - shortTermRefPicSetSpsFlag, err := bits.ReadFlag(buf, &pos) - if err != nil { - return 0, err - } - - var rps *SPS_ShortTermRefPicSet - - if !shortTermRefPicSetSpsFlag { - rps = &SPS_ShortTermRefPicSet{} - err = rps.unmarshal(buf, &pos, uint32(len(sps.ShortTermRefPicSets)), - uint32(len(sps.ShortTermRefPicSets)), sps.ShortTermRefPicSets) - if err != nil { - return 0, err - } - } else { - if len(sps.ShortTermRefPicSets) == 0 { - return 0, fmt.Errorf("invalid short_term_ref_pic_set_idx") - } - - b := int(math.Ceil(math.Log2(float64(len(sps.ShortTermRefPicSets))))) - tmp, err := bits.ReadBits(buf, &pos, b) - if err != nil { - return 0, err - } - shortTermRefPicSetIdx := int(tmp) - - if len(sps.ShortTermRefPicSets) <= shortTermRefPicSetIdx { - return 0, fmt.Errorf("invalid short_term_ref_pic_set_idx") - } - - rps = sps.ShortTermRefPicSets[shortTermRefPicSetIdx] - } - - var v uint32 - - if sliceType == 0 { // B-frame - if typ == NALUType_TRAIL_N || typ == NALUType_RASL_N { - v = sps.MaxNumReorderPics[0] - uint32(len(rps.DeltaPocS1)) - } else if typ == NALUType_TRAIL_R || typ == NALUType_RASL_R { - if len(rps.DeltaPocS0) == 0 { - return 0, fmt.Errorf("invalid DeltaPocS0") - } - v = uint32(-rps.DeltaPocS0[0]-1+int32(sps.MaxNumReorderPics[0])) - uint32(len(rps.DeltaPocS1)) - } - } else { // I or P-frame - if len(rps.DeltaPocS0) == 0 { - return 0, fmt.Errorf("invalid DeltaPocS0") - } - v = uint32(-rps.DeltaPocS0[0] - 1 + int32(sps.MaxNumReorderPics[0])) - } - - return v, nil -} - -// DTSExtractor allows to extract DTS from PTS. +// DTSExtractor computes DTS from PTS. +// +// Deprecated: replaced by DTSExtractor2. type DTSExtractor struct { spsp *SPS ppsp *PPS @@ -142,6 +16,8 @@ type DTSExtractor struct { } // NewDTSExtractor allocates a DTSExtractor. +// +// Deprecated: replaced by NewDTSExtractor2. func NewDTSExtractor() *DTSExtractor { return &DTSExtractor{} } diff --git a/pkg/codecs/h265/dts_extractor2.go b/pkg/codecs/h265/dts_extractor2.go new file mode 100644 index 0000000..40cadb3 --- /dev/null +++ b/pkg/codecs/h265/dts_extractor2.go @@ -0,0 +1,239 @@ +package h265 + +import ( + "fmt" + "math" + + "github.com/bluenviron/mediacommon/pkg/bits" + "github.com/bluenviron/mediacommon/pkg/codecs/h264" +) + +const ( + maxBytesToGetPOC = 12 +) + +func getPTSDTSDiff(buf []byte, sps *SPS, pps *PPS) (uint32, error) { + typ := NALUType((buf[0] >> 1) & 0b111111) + + buf = buf[1:] + lb := len(buf) + + if lb > maxBytesToGetPOC { + lb = maxBytesToGetPOC + } + + buf = h264.EmulationPreventionRemove(buf[:lb]) + pos := 8 + + firstSliceSegmentInPicFlag, err := bits.ReadFlag(buf, &pos) + if err != nil { + return 0, err + } + + if !firstSliceSegmentInPicFlag { + return 0, fmt.Errorf("first_slice_segment_in_pic_flag = 0 is not supported") + } + + if typ >= NALUType_BLA_W_LP && typ <= NALUType_RSV_IRAP_VCL23 { + _, err = bits.ReadFlag(buf, &pos) // no_output_of_prior_pics_flag + if err != nil { + return 0, err + } + } + + _, err = bits.ReadGolombUnsigned(buf, &pos) // slice_pic_parameter_set_id + if err != nil { + return 0, err + } + + if pps.NumExtraSliceHeaderBits > 0 { + err = bits.HasSpace(buf, pos, int(pps.NumExtraSliceHeaderBits)) + if err != nil { + return 0, err + } + pos += int(pps.NumExtraSliceHeaderBits) + } + + sliceType, err := bits.ReadGolombUnsigned(buf, &pos) // slice_type + if err != nil { + return 0, err + } + + if pps.OutputFlagPresentFlag { + _, err = bits.ReadFlag(buf, &pos) // pic_output_flag + if err != nil { + return 0, err + } + } + + if sps.SeparateColourPlaneFlag { + _, err = bits.ReadBits(buf, &pos, 2) // colour_plane_id + if err != nil { + return 0, err + } + } + + _, err = bits.ReadBits(buf, &pos, int(sps.Log2MaxPicOrderCntLsbMinus4+4)) // pic_order_cnt_lsb + if err != nil { + return 0, err + } + + shortTermRefPicSetSpsFlag, err := bits.ReadFlag(buf, &pos) + if err != nil { + return 0, err + } + + var rps *SPS_ShortTermRefPicSet + + if !shortTermRefPicSetSpsFlag { + rps = &SPS_ShortTermRefPicSet{} + err = rps.unmarshal(buf, &pos, uint32(len(sps.ShortTermRefPicSets)), + uint32(len(sps.ShortTermRefPicSets)), sps.ShortTermRefPicSets) + if err != nil { + return 0, err + } + } else { + if len(sps.ShortTermRefPicSets) == 0 { + return 0, fmt.Errorf("invalid short_term_ref_pic_set_idx") + } + + b := int(math.Ceil(math.Log2(float64(len(sps.ShortTermRefPicSets))))) + tmp, err := bits.ReadBits(buf, &pos, b) + if err != nil { + return 0, err + } + shortTermRefPicSetIdx := int(tmp) + + if len(sps.ShortTermRefPicSets) <= shortTermRefPicSetIdx { + return 0, fmt.Errorf("invalid short_term_ref_pic_set_idx") + } + + rps = sps.ShortTermRefPicSets[shortTermRefPicSetIdx] + } + + var v uint32 + + if sliceType == 0 { // B-frame + if typ == NALUType_TRAIL_N || typ == NALUType_RASL_N { + v = sps.MaxNumReorderPics[0] - uint32(len(rps.DeltaPocS1)) + } else if typ == NALUType_TRAIL_R || typ == NALUType_RASL_R { + if len(rps.DeltaPocS0) == 0 { + return 0, fmt.Errorf("invalid DeltaPocS0") + } + v = uint32(-rps.DeltaPocS0[0]-1+int32(sps.MaxNumReorderPics[0])) - uint32(len(rps.DeltaPocS1)) + } + } else { // I or P-frame + if len(rps.DeltaPocS0) == 0 { + return 0, fmt.Errorf("invalid DeltaPocS0") + } + v = uint32(-rps.DeltaPocS0[0] - 1 + int32(sps.MaxNumReorderPics[0])) + } + + return v, nil +} + +// DTSExtractor2 computes DTS from PTS. +type DTSExtractor2 struct { + spsp *SPS + ppsp *PPS + prevDTSFilled bool + prevDTS int64 +} + +// NewDTSExtractor2 allocates a DTSExtractor. +func NewDTSExtractor2() *DTSExtractor2 { + return &DTSExtractor2{} +} + +func (d *DTSExtractor2) extractInner(au [][]byte, pts int64) (int64, error) { + var idr []byte + var nonIDR []byte + + for _, nalu := range au { + typ := NALUType((nalu[0] >> 1) & 0b111111) + switch typ { + case NALUType_SPS_NUT: + var spsp SPS + err := spsp.Unmarshal(nalu) + if err != nil { + return 0, fmt.Errorf("invalid SPS: %w", err) + } + d.spsp = &spsp + + case NALUType_PPS_NUT: + var ppsp PPS + err := ppsp.Unmarshal(nalu) + if err != nil { + return 0, fmt.Errorf("invalid PPS: %w", err) + } + d.ppsp = &ppsp + + case NALUType_IDR_W_RADL, NALUType_IDR_N_LP: + idr = nalu + + case NALUType_TRAIL_N, NALUType_TRAIL_R, NALUType_CRA_NUT, NALUType_RASL_N, NALUType_RASL_R: + nonIDR = nalu + } + } + + if d.spsp == nil { + return 0, fmt.Errorf("SPS not received yet") + } + + if d.ppsp == nil { + return 0, fmt.Errorf("PPS not received yet") + } + + if len(d.spsp.MaxNumReorderPics) != 1 || d.spsp.MaxNumReorderPics[0] == 0 { + return pts, nil + } + + if d.spsp.VUI == nil || d.spsp.VUI.TimingInfo == nil { + return pts, nil + } + + var samplesDiff uint32 + + switch { + case idr != nil: + samplesDiff = d.spsp.MaxNumReorderPics[0] + + case nonIDR != nil: + var err error + samplesDiff, err = getPTSDTSDiff(nonIDR, d.spsp, d.ppsp) + if err != nil { + return 0, err + } + + default: + return 0, fmt.Errorf("access unit doesn't contain an IDR or non-IDR NALU") + } + + timeDiff := int64(samplesDiff) * 90000 * + int64(d.spsp.VUI.TimingInfo.NumUnitsInTick) / int64(d.spsp.VUI.TimingInfo.TimeScale) + dts := pts - timeDiff + + return dts, nil +} + +// Extract extracts the DTS of a access unit. +func (d *DTSExtractor2) Extract(au [][]byte, pts int64) (int64, error) { + dts, err := d.extractInner(au, pts) + if err != nil { + return 0, err + } + + if dts > pts { + return 0, fmt.Errorf("DTS is greater than PTS") + } + + if d.prevDTSFilled && dts < d.prevDTS { + return 0, fmt.Errorf("DTS is not monotonically increasing, was %v, now is %v", + d.prevDTS, dts) + } + + d.prevDTSFilled = true + d.prevDTS = dts + + return dts, err +} diff --git a/pkg/codecs/h265/dts_extractor2_test.go b/pkg/codecs/h265/dts_extractor2_test.go new file mode 100644 index 0000000..23f6f2f --- /dev/null +++ b/pkg/codecs/h265/dts_extractor2_test.go @@ -0,0 +1,365 @@ +package h265 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDTSExtractor2(t *testing.T) { + type sequenceSample struct { + au [][]byte + pts int64 + dts int64 + } + + for _, ca := range []struct { + name string + sequence []sequenceSample + }{ + { + "with timing info, IDR", + []sequenceSample{ + { + [][]byte{ + { // VPS + 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, + 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x03, 0x00, 0x78, 0x99, 0x98, 0x09, + }, + { // SPS + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, + 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, + 0x00, 0x78, 0xa0, 0x03, 0xc0, 0x80, 0x10, 0xe5, + 0x96, 0x66, 0x69, 0x24, 0xca, 0xe0, 0x10, 0x00, + 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, 0x01, + 0xe0, 0x80, + }, + { // PPS + 0x44, 0x1, 0xc1, 0x72, 0xb4, 0x62, 0x40, + }, + { // IDR_W_RADL + 0x26, 0x1, 0xaf, 0x8, 0x42, 0x23, 0x48, 0x8a, 0x43, 0xe2, + }, + }, + 0, + -6000, + }, + { + [][]byte{{ // TRAIL_R + 0x02, 0x01, 0xd0, 0x19, 0x5f, 0x8c, 0xb4, 0x42, + 0x49, 0x20, 0x40, 0x11, 0x16, 0x92, 0x93, 0xea, + 0x54, 0x57, 0x4e, 0x0a, + }}, + int64(100 * time.Millisecond * 90000 / time.Second), + -3000, + }, + { + [][]byte{{ // TRAIL_R + 0x02, 0x01, 0xe0, 0x44, 0x97, 0xe0, 0x81, 0x20, + 0x44, 0x52, 0x62, 0x7a, 0x1b, 0x88, 0x0b, 0x21, + 0x26, 0x5f, 0x10, 0x9c, + }}, + int64(66666666 * time.Nanosecond * 90000 / time.Second), + -1, + }, + { + [][]byte{{ // TRAIL_N + 0x00, 0x01, 0xe0, 0x24, 0xff, 0xfa, 0x24, 0x0a, + 0x42, 0x25, 0x8c, 0x18, 0xe6, 0x1c, 0xea, 0x5a, + 0x5d, 0x07, 0xc1, 0x8f, + }}, + int64(33333333 * time.Nanosecond * 90000 / time.Second), + int64(33333333 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{ // TRAIL_R + 0x02, 0x01, 0xd0, 0x30, 0x97, 0xd7, 0xdc, 0xf9, + 0x0c, 0x10, 0x11, 0x11, 0x20, 0x42, 0x11, 0x18, + 0x63, 0xa5, 0x18, 0x55, + }}, + int64(200 * time.Millisecond * 90000 / time.Second), + int64(66666667 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{{ // TRAIL_R + 0x02, 0x01, 0xe0, 0xa2, 0x25, 0xd7, 0xf7, 0x08, + 0x12, 0x04, 0x45, 0xa1, 0x83, 0xc0, 0x97, 0x53, + 0xa3, 0x5e, 0x78, 0x14, + }}, + int64(166666666 * time.Nanosecond * 90000 / time.Second), + 8999, + }, + { + [][]byte{{ // TRAIL_N + 0x00, 0x01, 0xe0, 0x82, 0x3f, 0x5f, 0xf6, 0x89, + 0x02, 0x90, 0x88, 0xa3, 0x0c, 0x7d, 0x27, 0x0c, + 0xd4, 0xd9, 0xc2, 0xa5, + }}, + int64(133333333 * time.Nanosecond * 90000 / time.Second), + int64(133333333 * time.Nanosecond * 90000 / time.Second), + }, + }, + }, + { + "no timing info, CRA", + []sequenceSample{ + { + [][]byte{ + { // SPS + 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, + 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, + 0x00, 0x7b, 0xa0, 0x07, 0x82, 0x00, 0x88, 0x7d, + 0xb6, 0x71, 0x8b, 0x92, 0x44, 0x80, 0x53, 0x88, + 0x88, 0x92, 0xcf, 0x24, 0xa6, 0x92, 0x72, 0xc9, + 0x12, 0x49, 0x22, 0xdc, 0x91, 0xaa, 0x48, 0xfc, + 0xa2, 0x23, 0xff, 0x00, 0x01, 0x00, 0x01, 0x6a, + 0x02, 0x02, 0x02, 0x01, + }, + { // PPS + 0x44, 0x01, 0xc0, 0x25, 0x2f, 0x05, 0x32, 0x40, + }, + { + byte(NALUType_CRA_NUT) << 1, + }, + }, + int64(1 * time.Second * 90000 / time.Second), + int64(1 * time.Second * 90000 / time.Second), + }, + }, + }, + { + "short_term_ref_pic_set_sps_flag", + []sequenceSample{ + { + [][]byte{ + { // VPS + 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40, + 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x03, 0x00, 0x99, 0xa5, 0x02, 0x40, + }, + { // SPS + 0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03, + 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, + 0x00, 0x99, 0xa0, 0x03, 0xc0, 0x80, 0x10, 0xe5, + 0x8d, 0xa5, 0x92, 0x42, 0x36, 0x22, 0xec, 0xb8, + 0x80, 0x40, 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, + 0x00, 0x05, 0x0f, 0xe2, 0xc4, 0xa0, + }, + { // PPS + 0x44, 0x01, 0xc0, 0xe0, 0x98, 0x93, 0x03, 0x05, + 0x14, 0x90, + }, + { // IDR + 0x26, 0x01, 0xaf, 0x3e, 0x3d, 0x3a, 0xca, 0xc0, + 0xf2, 0x2f, 0xc3, 0x0f, 0x86, 0x9f, 0xed, 0xfc, + 0x67, 0x2f, 0x62, 0x69, + }, + }, + int64(1113436 * time.Nanosecond * 90000 / time.Second), + -4400, + }, + { + [][]byte{ + { // TRAIL_R + 0x02, 0x02, 0xd0, 0x00, 0x0c, 0xc6, 0x27, 0xfe, + 0x6e, 0x6d, 0xe8, 0x10, 0xd5, 0xce, 0x61, 0x1b, + 0x66, 0xf6, 0x21, 0x59, + }, + }, + int64(68113436 * time.Nanosecond * 90000 / time.Second), + 1630, + }, + { + [][]byte{ + { // TRAIL_R + 0x02, 0x02, 0xd0, 0x00, 0x14, 0xc6, 0x7c, 0xfe, + 0x83, 0x29, 0x34, 0xba, 0xce, 0xaa, 0x8b, 0x76, + 0xb0, 0x95, 0x67, 0xb2, + }, + }, + int64(101113436 * time.Nanosecond * 90000 / time.Second), + int64(51113436 * time.Nanosecond * 90000 / time.Second), + }, + }, + }, + { + "qsv hevc b_frames=3", + []sequenceSample{ + { + [][]byte{ //nolint:dupl + { // VPS + 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40, + 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x03, 0x00, 0x7b, 0x11, 0xc0, 0xc0, + 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x0f, + 0x14, + }, + { // SPS + 0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03, + 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, + 0x00, 0x7b, 0xa0, 0x03, 0xc0, 0x80, 0x11, 0x07, + 0xcb, 0xb1, 0x1e, 0xe4, 0x6c, 0x0a, 0x9f, 0xa6, + 0xb9, 0x97, 0x92, 0xcf, 0x60, 0x2d, 0x40, 0x40, + 0x40, 0x45, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, + 0x00, 0x03, 0x00, 0x3c, 0x60, 0x35, 0xef, 0x7e, + 0x00, 0x02, 0x62, 0x58, 0x00, 0x26, 0x17, 0x20, + }, + { // PPS + 0x44, 0x01, 0xc0, 0x3c, 0xf0, 0x1b, 0x64, + }, + { + 0x4e, 0x01, 0x00, 0x0a, 0x80, 0x00, 0x00, 0x03, + 0x00, 0x55, 0xda, 0x80, 0x01, 0xe5, 0xd0, 0x01, + 0x07, 0x04, 0x00, 0x00, 0xee, 0x00, 0x00, 0x05, + 0x80, + }, + { + 0x26, 0x01, 0xae, 0x80, 0x8f, 0x4c, 0xdd, 0xfc, + 0xee, 0x2f, 0x79, 0x7c, 0x9e, 0x21, 0x6b, 0x2a, + 0xe7, 0x6a, 0x57, 0x56, 0x46, 0x6f, 0x32, 0x5a, + 0x7c, 0xbc, 0x47, 0xe8, 0xce, 0x5c, 0x5e, 0xfa, + 0x1e, 0xd0, 0x94, 0x08, 0x4c, 0x98, 0x9d, 0xbb, + 0x5d, 0x4c, 0x54, 0xa1, 0xd9, 0x5b, 0x1b, 0xba, + }, + }, + int64(2033333333 * time.Nanosecond * 90000 / time.Second), + 179999, + }, + { + [][]byte{ + { + 0x4e, 0x01, 0x01, 0x07, 0x04, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x03, 0x00, 0x0b, 0x80, + }, + { + 0x02, 0x01, 0xe2, 0x0a, 0x4f, 0xdd, 0x1e, 0xb7, + 0xb7, 0xa1, 0x80, 0xad, 0xc7, 0x3c, 0x2e, 0x33, + 0x3b, 0xde, 0xcc, 0x77, 0x13, 0x9c, 0x5b, 0xe3, + 0x2c, 0xaa, 0xd4, 0x2e, 0xb0, 0x2b, 0x9e, 0x20, + 0xdd, 0xc9, 0x1b, 0x39, 0xd9, 0x75, 0x06, 0xf5, + 0xa8, 0x1f, 0x66, 0x62, 0x5b, 0xfe, 0x1f, 0xf9, + }, + }, + int64(2099999999 * time.Nanosecond * 90000 / time.Second), + int64(2016666666 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{ + { + 0x4e, 0x01, 0x01, 0x07, 0x04, 0x00, 0x00, 0x03, + 0x02, 0x00, 0x00, 0x05, 0x80, + }, + { + 0x02, 0x01, 0xe1, 0x32, 0x27, 0xe3, 0xa0, 0x51, + 0xcd, 0xff, 0x1a, 0x0b, 0x37, 0xaf, 0xe3, 0xe6, + 0x9e, 0xaa, 0x27, 0x82, 0xcd, 0x28, 0xa3, 0xce, + 0x57, 0x8b, 0x02, 0x3e, 0x62, 0x1f, 0x66, 0x5b, + 0xbd, 0x67, 0x6b, 0xb1, 0x47, 0x9d, 0x1b, 0x26, + 0xb7, 0x2a, 0x04, 0xac, 0x2e, 0x94, 0x1e, 0x22, + }, + }, + int64(2066666666 * time.Nanosecond * 90000 / time.Second), + int64(2033333333 * time.Nanosecond * 90000 / time.Second), + }, + { + [][]byte{ + { + 0x4e, 0x01, 0x01, 0x07, 0x04, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x03, 0x01, 0x80, + }, + { + 0x00, 0x01, 0xe0, 0xcf, 0x8e, 0x80, 0x1e, 0x96, + 0xa6, 0x88, 0xcb, 0x98, 0xf3, 0xd9, 0x2a, 0x4b, + 0xa7, 0xa0, 0xf8, 0xa0, 0x4d, 0x21, 0x89, 0x76, + 0x54, 0xe3, 0x8f, 0x46, 0xf6, 0x93, 0xde, 0x84, + 0x33, 0x26, 0x3e, 0xe8, 0x20, 0x23, 0xef, 0x39, + 0x03, 0x3c, 0x92, 0x11, 0x50, 0x98, 0xd6, 0x13, + }, + }, + int64(2049999999 * time.Nanosecond * 90000 / time.Second), + int64(2049999999 * time.Nanosecond * 90000 / time.Second), + }, + }, + }, + } { + t.Run(ca.name, func(t *testing.T) { + ex := NewDTSExtractor2() + for _, sample := range ca.sequence { + dts, err := ex.Extract(sample.au, sample.pts) + require.NoError(t, err) + require.Equal(t, sample.dts, dts) + } + }) + } +} + +func FuzzDTSExtractor2FirstAU(f *testing.F) { + f.Fuzz(func(_ *testing.T, a []byte, b []byte, c []byte) { + if len(a) < 1 || len(b) < 1 || len(c) < 1 { + return + } + + ex := NewDTSExtractor() + + ex.Extract([][]byte{ //nolint:errcheck + a, + b, + c, + }, 0) + }) +} + +func FuzzDTSExtractor2SecondAU(f *testing.F) { //nolint:dupl + f.Fuzz(func(t *testing.T, sps int, a []byte) { + if len(a) < 1 { + return + } + + ex := NewDTSExtractor() + + switch sps % 2 { + case 0: + _, err := ex.Extract([][]byte{ + { // SPS + 0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03, + 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, + 0x00, 0x99, 0xa0, 0x03, 0xc0, 0x80, 0x10, 0xe5, + 0x8d, 0xa5, 0x92, 0x42, 0x36, 0x22, 0xec, 0xb8, + 0x80, 0x40, 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, + 0x00, 0x05, 0x0f, 0xe2, 0xc4, 0xa0, + }, + { // PPS + 0x44, 0x01, 0xc0, 0xe0, 0x98, 0x93, 0x03, 0x05, + 0x14, 0x90, + }, + { // IDR + 0x26, + }, + }, 0) + require.NoError(t, err) + + default: + _, err := ex.Extract([][]byte{ + { // SPS + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, + 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, + 0x00, 0x78, 0xa0, 0x03, 0xc0, 0x80, 0x10, 0xe5, + 0x96, 0x66, 0x69, 0x24, 0xca, 0xe0, 0x10, 0x00, + 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, 0x01, + 0xe0, 0x80, + }, + { // PPS + 0x44, 0x01, 0xc1, 0x72, 0xb4, 0x62, 0x40, + }, + { // IDR + 0x26, + }, + }, 0) + require.NoError(t, err) + } + + ex.Extract([][]byte{a}, 1*time.Second) //nolint:errcheck + }) +} diff --git a/pkg/codecs/h265/dts_extractor_test.go b/pkg/codecs/h265/dts_extractor_test.go index 8ed5497..ab27deb 100644 --- a/pkg/codecs/h265/dts_extractor_test.go +++ b/pkg/codecs/h265/dts_extractor_test.go @@ -188,7 +188,7 @@ func TestDTSExtractor(t *testing.T) { "qsv hevc b_frames=3", []sequenceSample{ { - [][]byte{ + [][]byte{ //nolint:dupl { // VPS 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, @@ -311,7 +311,7 @@ func FuzzDTSExtractorFirstAU(f *testing.F) { }) } -func FuzzDTSExtractorSecondAU(f *testing.F) { +func FuzzDTSExtractorSecondAU(f *testing.F) { //nolint:dupl f.Fuzz(func(t *testing.T, sps int, a []byte) { if len(a) < 1 { return diff --git a/pkg/formats/mpegts/reader.go b/pkg/formats/mpegts/reader.go index d9b67d4..076f289 100644 --- a/pkg/formats/mpegts/reader.go +++ b/pkg/formats/mpegts/reader.go @@ -110,7 +110,7 @@ func (r *Reader) OnDecodeError(cb ReaderOnDecodeErrorFunc) { // OnDataH26x sets a callback that is called when data from an H265 or H264 track is received. // -// Deprecated: replaced by OnDataH264, OnDataH265 +// Deprecated: replaced by OnDataH264, OnDataH265. func (r *Reader) OnDataH26x(track *Track, cb ReaderOnDataH26xFunc) { if _, ok := track.Codec.(*CodecH265); ok { r.OnDataH265(track, cb) diff --git a/pkg/formats/mpegts/time_decoder.go b/pkg/formats/mpegts/time_decoder.go index 69034c5..170fd54 100644 --- a/pkg/formats/mpegts/time_decoder.go +++ b/pkg/formats/mpegts/time_decoder.go @@ -18,14 +18,14 @@ func multiplyAndDivide(v, m, d time.Duration) time.Duration { // TimeDecoder is a MPEG-TS timestamp decoder. // -// Deprecated: replaced by TimeDecoder2 +// Deprecated: replaced by TimeDecoder2. type TimeDecoder struct { wrapped *TimeDecoder2 } // NewTimeDecoder allocates a TimeDecoder. // -// Deprecated: replaced by NewTimeDecoder2 +// Deprecated: replaced by NewTimeDecoder2. func NewTimeDecoder(start int64) *TimeDecoder { return &TimeDecoder{ wrapped: NewTimeDecoder2(start),