From d168642ae0e5d2fa7870556018acc3bf58c9c0fe Mon Sep 17 00:00:00 2001 From: tonihei Date: Wed, 20 Mar 2024 04:55:06 -0700 Subject: [PATCH] Don't set codec color info for default SDR Some media can read color info values from the bitstream and may partially set some of the SDR default values in Format.ColorInfo. Setting these default values for SDR can confuse some codecs and may also prevent adaptive ABR switches if not all ColorInfo values are set in exactly the same way. We can avoid any influence of HDR color info handling by disabling setting the color info MediaFormat keys for SDR video and also avoid codec reset at format changes if both formats are SDR with slightly different ColorInfo settings. To identify "SDR" ColorInfo instances, we need to do some fuzzy matching as many of the default values are assumed to match the SDR profile even if not set. Issue: androidx/media#1158 PiperOrigin-RevId: 617473937 --- .../exoplayer2/util/MediaFormatUtil.java | 2 +- .../android/exoplayer2/video/ColorInfo.java | 42 +++++++- .../exoplayer2/util/MediaFormatUtilTest.java | 14 +++ .../exoplayer2/mediacodec/MediaCodecInfo.java | 6 +- .../mediacodec/MediaCodecInfoTest.java | 96 +++++++++++++++---- 5 files changed, 132 insertions(+), 28 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/MediaFormatUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/MediaFormatUtil.java index fce61fa9d07..2f26495d427 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/MediaFormatUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/MediaFormatUtil.java @@ -258,7 +258,7 @@ public static void maybeSetByteBuffer(MediaFormat format, String key, @Nullable */ @SuppressWarnings("InlinedApi") public static void maybeSetColorInfo(MediaFormat format, @Nullable ColorInfo colorInfo) { - if (colorInfo != null) { + if (!ColorInfo.isEquivalentToAssumedSdrDefault(colorInfo)) { maybeSetInteger(format, MediaFormat.KEY_COLOR_TRANSFER, colorInfo.colorTransfer); maybeSetInteger(format, MediaFormat.KEY_COLOR_STANDARD, colorInfo.colorSpace); maybeSetInteger(format, MediaFormat.KEY_COLOR_RANGE, colorInfo.colorRange); diff --git a/library/common/src/main/java/com/google/android/exoplayer2/video/ColorInfo.java b/library/common/src/main/java/com/google/android/exoplayer2/video/ColorInfo.java index bd7bbb88ba7..40807d93665 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/video/ColorInfo.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/video/ColorInfo.java @@ -23,6 +23,7 @@ import com.google.android.exoplayer2.util.Util; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.Arrays; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.dataflow.qual.Pure; /** @@ -179,6 +180,34 @@ public ColorInfo build() { .setColorTransfer(C.COLOR_TRANSFER_SRGB) .build(); + /** + * Returns whether the given color info is equivalent to values for a standard dynamic range video + * that could generally be assumed if no further information is given. + * + *

The color info is deemed to be equivalent to SDR video if it either has unset values or + * values matching a 8-bit (chroma+luma), BT.709 or BT.601 color space, SDR transfer and Limited + * range color info. + * + * @param colorInfo The color info to evaluate. + * @return Whether the given color info is equivalent to the assumed default SDR color info. + */ + @EnsuresNonNullIf(result = false, expression = "#1") + public static boolean isEquivalentToAssumedSdrDefault(@Nullable ColorInfo colorInfo) { + if (colorInfo == null) { + return true; + } + return (colorInfo.colorSpace == Format.NO_VALUE + || colorInfo.colorSpace == C.COLOR_SPACE_BT709 + || colorInfo.colorSpace == C.COLOR_SPACE_BT601) + && (colorInfo.colorRange == Format.NO_VALUE + || colorInfo.colorRange == C.COLOR_RANGE_LIMITED) + && (colorInfo.colorTransfer == Format.NO_VALUE + || colorInfo.colorTransfer == C.COLOR_TRANSFER_SDR) + && colorInfo.hdrStaticInfo == null + && (colorInfo.chromaBitdepth == Format.NO_VALUE || colorInfo.chromaBitdepth == 8) + && (colorInfo.lumaBitdepth == Format.NO_VALUE || colorInfo.lumaBitdepth == 8); + } + /** * Returns the {@link C.ColorSpace} corresponding to the given ISO color primary code, as per * table A.7.21.1 in Rec. ITU-T T.832 (03/2009), or {@link Format#NO_VALUE} if no mapping can be @@ -239,22 +268,25 @@ public static boolean isTransferHdr(@Nullable ColorInfo colorInfo) { || colorInfo.colorTransfer == C.COLOR_TRANSFER_ST2084); } - /** The {@link C.ColorSpace}. */ + /** The {@link C.ColorSpace}, or {@link Format#NO_VALUE} if not set. */ public final @C.ColorSpace int colorSpace; - /** The {@link C.ColorRange}. */ + /** The {@link C.ColorRange}, or {@link Format#NO_VALUE} if not set. */ public final @C.ColorRange int colorRange; - /** The {@link C.ColorTransfer}. */ + /** The {@link C.ColorTransfer}, or {@link Format#NO_VALUE} if not set. */ public final @C.ColorTransfer int colorTransfer; /** HdrStaticInfo as defined in CTA-861.3, or null if none specified. */ @Nullable public final byte[] hdrStaticInfo; - /** The bit depth of the luma samples of the video. */ + /** The bit depth of the luma samples of the video, or {@link Format#NO_VALUE} if not set. */ public final int lumaBitdepth; - /** The bit depth of the chroma samples of the video. It may differ from the luma bit depth. */ + /** + * The bit depth of the chroma samples of the video, or {@link Format#NO_VALUE} if not set. It may + * differ from the luma bit depth. + */ public final int chromaBitdepth; // Lazily initialized hashcode. diff --git a/library/common/src/test/java/com/google/android/exoplayer2/util/MediaFormatUtilTest.java b/library/common/src/test/java/com/google/android/exoplayer2/util/MediaFormatUtilTest.java index 8707a060a8c..ad3cd05e987 100644 --- a/library/common/src/test/java/com/google/android/exoplayer2/util/MediaFormatUtilTest.java +++ b/library/common/src/test/java/com/google/android/exoplayer2/util/MediaFormatUtilTest.java @@ -232,9 +232,23 @@ public void createMediaFormatFromFormat_withPopulatedFormat_generatesExpectedEnt @Test public void createMediaFormatFromFormat_withCustomPcmEncoding_setsCustomPcmEncodingEntry() { Format format = new Format.Builder().setPcmEncoding(C.ENCODING_PCM_16BIT_BIG_ENDIAN).build(); + MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format); + assertThat(mediaFormat.getInteger(MediaFormatUtil.KEY_PCM_ENCODING_EXTENDED)) .isEqualTo(C.ENCODING_PCM_16BIT_BIG_ENDIAN); assertThat(mediaFormat.containsKey(MediaFormat.KEY_PCM_ENCODING)).isFalse(); } + + @Test + public void createMediaFormatFromFormat_withSdrColorInfo_omitsMediaFormatColorInfoKeys() { + Format format = new Format.Builder().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build(); + + MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format); + + assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)).isFalse(); + assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_RANGE)).isFalse(); + assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_STANDARD)).isFalse(); + assertThat(mediaFormat.containsKey(MediaFormat.KEY_HDR_STATIC_INFO)).isFalse(); + } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java index b27c6ed1ba1..1d1ae55eee6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java @@ -49,6 +49,7 @@ import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.ColorInfo; /** * Information about a {@link MediaCodec} for a given MIME type. @@ -428,7 +429,10 @@ public DecoderReuseEvaluation canReuseCodec(Format oldFormat, Format newFormat) && (oldFormat.width != newFormat.width || oldFormat.height != newFormat.height)) { discardReasons |= DISCARD_REASON_VIDEO_RESOLUTION_CHANGED; } - if (!Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo)) { + if ((!ColorInfo.isEquivalentToAssumedSdrDefault(oldFormat.colorInfo) + || !ColorInfo.isEquivalentToAssumedSdrDefault(newFormat.colorInfo)) + && !Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo)) { + // Don't perform detailed checks if both ColorInfos fall within the default SDR assumption. discardReasons |= DISCARD_REASON_VIDEO_COLOR_INFO_CHANGED; } if (needsAdaptationReconfigureWorkaround(name) diff --git a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfoTest.java b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfoTest.java index d79f19e5b8d..339ebfcd47e 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfoTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfoTest.java @@ -74,7 +74,7 @@ public final class MediaCodecInfoTest { .build(); @Test - public void canKeepCodec_withDifferentMimeType_returnsNo() { + public void canReuseCodec_withDifferentMimeType_returnsNo() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true); Format hdAv1Format = FORMAT_H264_HD.buildUpon().setSampleMimeType(VIDEO_AV1).build(); @@ -89,7 +89,7 @@ public void canKeepCodec_withDifferentMimeType_returnsNo() { } @Test - public void canKeepCodec_withRotation_returnsNo() { + public void canReuseCodec_withRotation_returnsNo() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true); Format hdRotatedFormat = FORMAT_H264_HD.buildUpon().setRotationDegrees(90).build(); @@ -104,7 +104,7 @@ public void canKeepCodec_withRotation_returnsNo() { } @Test - public void canKeepCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconfiguration() { + public void canReuseCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconfiguration() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true); assertThat(codecInfo.canReuseCodec(FORMAT_H264_HD, FORMAT_H264_4K)) @@ -118,7 +118,7 @@ public void canKeepCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconf } @Test - public void canKeepCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() { + public void canReuseCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); assertThat(codecInfo.canReuseCodec(FORMAT_H264_HD, FORMAT_H264_4K)) @@ -132,7 +132,7 @@ public void canKeepCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() { } @Test - public void canKeepCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithReconfiguration() { + public void canReuseCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithReconfiguration() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdVariantFormat = @@ -148,11 +148,11 @@ public void canKeepCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithRecon } @Test - public void canKeepCodec_colorInfoOmittedFromNewFormat_returnsNo() { + public void canReuseCodec_hdrToSdr_returnsNo() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); assertThat(codecInfo.canReuseCodec(hdrVariantFormat, FORMAT_H264_4K)) .isEqualTo( new DecoderReuseEvaluation( @@ -164,11 +164,11 @@ public void canKeepCodec_colorInfoOmittedFromNewFormat_returnsNo() { } @Test - public void canKeepCodec_colorInfoOmittedFromOldFormat_returnsNo() { + public void canReuseCodec_sdrToHdr_returnsNo() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); assertThat(codecInfo.canReuseCodec(FORMAT_H264_4K, hdrVariantFormat)) .isEqualTo( new DecoderReuseEvaluation( @@ -180,13 +180,13 @@ public void canKeepCodec_colorInfoOmittedFromOldFormat_returnsNo() { } @Test - public void canKeepCodec_colorInfoChange_returnsNo() { + public void canReuseCodec_hdrColorInfoChange_returnsNo() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat1 = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); Format hdrVariantFormat2 = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT709)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT709)).build(); assertThat(codecInfo.canReuseCodec(hdrVariantFormat1, hdrVariantFormat2)) .isEqualTo( new DecoderReuseEvaluation( @@ -198,7 +198,61 @@ public void canKeepCodec_colorInfoChange_returnsNo() { } @Test - public void canKeepCodec_audioWithDifferentChannelCounts_returnsNo() { + public void canReuseCodec_nullColorInfoToSdr_returnsYesWithoutReconfiguration() { + MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); + + Format variantWithColorInfo = + FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build(); + assertThat(codecInfo.canReuseCodec(FORMAT_H264_4K, variantWithColorInfo)) + .isEqualTo( + new DecoderReuseEvaluation( + codecInfo.name, + FORMAT_H264_4K, + variantWithColorInfo, + DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION, + /* discardReasons= */ 0)); + } + + @Test + public void canReuseCodec_sdrToNullColorInfo_returnsYesWithoutReconfiguration() { + MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); + + Format variantWithColorInfo = + FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build(); + assertThat(codecInfo.canReuseCodec(variantWithColorInfo, FORMAT_H264_4K)) + .isEqualTo( + new DecoderReuseEvaluation( + codecInfo.name, + variantWithColorInfo, + FORMAT_H264_4K, + DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION, + /* discardReasons= */ 0)); + } + + @Test + public void canReuseCodec_sdrToSdrWithPartialInformation_returnsYesWithoutReconfiguration() { + MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); + + Format variantWithFullColorInfo = + FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build(); + Format variantWithPartialColorInfo = + FORMAT_H264_4K + .buildUpon() + .setColorInfo( + ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(Format.NO_VALUE).build()) + .build(); + assertThat(codecInfo.canReuseCodec(variantWithFullColorInfo, variantWithPartialColorInfo)) + .isEqualTo( + new DecoderReuseEvaluation( + codecInfo.name, + variantWithFullColorInfo, + variantWithPartialColorInfo, + DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION, + /* discardReasons= */ 0)); + } + + @Test + public void canReuseCodec_audioWithDifferentChannelCounts_returnsNo() { MediaCodecInfo codecInfo = buildAacCodecInfo(); assertThat(codecInfo.canReuseCodec(FORMAT_AAC_STEREO, FORMAT_AAC_SURROUND)) @@ -212,7 +266,7 @@ public void canKeepCodec_audioWithDifferentChannelCounts_returnsNo() { } @Test - public void canKeepCodec_audioWithSameChannelCounts_returnsYesWithFlush() { + public void canReuseCodec_audioWithSameChannelCounts_returnsYesWithFlush() { MediaCodecInfo codecInfo = buildAacCodecInfo(); Format stereoVariantFormat = FORMAT_AAC_STEREO.buildUpon().setAverageBitrate(100).build(); @@ -227,7 +281,7 @@ public void canKeepCodec_audioWithSameChannelCounts_returnsYesWithFlush() { } @Test - public void canKeepCodec_audioWithDifferentInitializationData_returnsNo() { + public void canReuseCodec_audioWithDifferentInitializationData_returnsNo() { MediaCodecInfo codecInfo = buildAacCodecInfo(); Format stereoVariantFormat = @@ -310,7 +364,7 @@ public void isSeamlessAdaptationSupported_colorInfoOmittedFromCompleteNewFormat_ MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); assertThat( codecInfo.isSeamlessAdaptationSupported( hdrVariantFormat, FORMAT_H264_4K, /* isNewFormatComplete= */ true)) @@ -323,7 +377,7 @@ public void isSeamlessAdaptationSupported_colorInfoOmittedFromIncompleteNewForma MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); assertThat( codecInfo.isSeamlessAdaptationSupported( hdrVariantFormat, FORMAT_H264_4K, /* isNewFormatComplete= */ false)) @@ -336,7 +390,7 @@ public void isSeamlessAdaptationSupported_colorInfoOmittedFromOldFormat_returnsF MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); assertThat( codecInfo.isSeamlessAdaptationSupported( FORMAT_H264_4K, hdrVariantFormat, /* isNewFormatComplete= */ true)) @@ -349,9 +403,9 @@ public void isSeamlessAdaptationSupported_colorInfoChange_returnsFalse() { MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false); Format hdrVariantFormat1 = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build(); Format hdrVariantFormat2 = - FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT709)).build(); + FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT709)).build(); assertThat( codecInfo.isSeamlessAdaptationSupported( hdrVariantFormat1, hdrVariantFormat2, /* isNewFormatComplete= */ true)) @@ -429,7 +483,7 @@ private static MediaCodecInfo buildAacCodecInfo() { /* secure= */ false); } - private static ColorInfo buildColorInfo(@C.ColorSpace int colorSpace) { + private static ColorInfo buildHdrColorInfo(@C.ColorSpace int colorSpace) { return new ColorInfo.Builder() .setColorSpace(colorSpace) .setColorRange(C.COLOR_RANGE_FULL)