Skip to content

Commit

Permalink
Mark output sample as decode-only based on start time
Browse files Browse the repository at this point in the history 
We currently do the same check on the input timestamps and
expect the output timestamps to match. Some codecs produce
samples with modified timestamps and the logic is a lot safer
when the comparison with the start time is done on the output
side of the codec.

Issue: #11000
PiperOrigin-RevId: 540228209
  • Loading branch information
@tonihei @icbaker
tonihei authored and icbaker committed Jun 14, 2023
1 parent 8952ef2 commit cf02581
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 23 deletions.
2 changes: 1 addition & 1 deletion library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public final void enable(
state = STATE_ENABLED;
onEnabled(joining, mayRenderStartOfStream);
replaceStream(formats, stream, startPositionUs, offsetUs);
resetPosition(positionUs, joining);
resetPosition(startPositionUs, joining);
}

@Override
Expand Down
22 changes: 1 addition & 21 deletions library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;

/** An abstract renderer that uses {@link MediaCodec} to decode samples for rendering. */
Expand Down Expand Up @@ -298,7 +297,6 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
private final DecoderInputBuffer buffer;
private final DecoderInputBuffer bypassSampleBuffer;
private final BatchBuffer bypassBatchBuffer;
private final ArrayList<Long> decodeOnlyPresentationTimestamps;
private final MediaCodec.BufferInfo outputBufferInfo;
private final ArrayDeque<OutputStreamInfo> pendingOutputStreamChanges;
private final OggOpusAudioPacketizer oggOpusAudioPacketizer;
Expand Down Expand Up @@ -399,7 +397,6 @@ public MediaCodecRenderer(
buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
bypassSampleBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
bypassBatchBuffer = new BatchBuffer();
decodeOnlyPresentationTimestamps = new ArrayList<>();
outputBufferInfo = new MediaCodec.BufferInfo();
currentPlaybackSpeed = 1f;
targetPlaybackSpeed = 1f;
Expand Down Expand Up @@ -919,7 +916,6 @@ protected void resetCodecStateForFlush() {
shouldSkipAdaptationWorkaroundOutputBuffer = false;
isDecodeOnlyOutputBuffer = false;
isLastOutputBuffer = false;
decodeOnlyPresentationTimestamps.clear();
largestQueuedPresentationTimeUs = C.TIME_UNSET;
lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
lastProcessedOutputBufferTimeUs = C.TIME_UNSET;
Expand Down Expand Up @@ -1376,9 +1372,6 @@ private boolean feedInputBuffer() throws ExoPlaybackException {
c2Mp3TimestampTracker.getLastOutputBufferPresentationTimeUs(inputFormat));
}

if (buffer.isDecodeOnly()) {
decodeOnlyPresentationTimestamps.add(presentationTimeUs);
}
if (waitingForFirstSampleInFormat) {
if (!pendingOutputStreamChanges.isEmpty()) {
pendingOutputStreamChanges.peekLast().formatQueue.add(presentationTimeUs, inputFormat);
Expand Down Expand Up @@ -1908,7 +1901,7 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
&& largestQueuedPresentationTimeUs != C.TIME_UNSET) {
outputBufferInfo.presentationTimeUs = largestQueuedPresentationTimeUs;
}
isDecodeOnlyOutputBuffer = isDecodeOnlyBuffer(outputBufferInfo.presentationTimeUs);
isDecodeOnlyOutputBuffer = outputBufferInfo.presentationTimeUs < getLastResetPositionUs();
isLastOutputBuffer =
lastBufferInStreamPresentationTimeUs == outputBufferInfo.presentationTimeUs;
updateOutputFormatForTime(outputBufferInfo.presentationTimeUs);
Expand Down Expand Up @@ -2199,19 +2192,6 @@ private void reinitializeCodec() throws ExoPlaybackException {
maybeInitCodecOrBypass();
}

private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
// We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
// box presentationTimeUs, creating a Long object that would need to be garbage collected.
int size = decodeOnlyPresentationTimestamps.size();
for (int i = 0; i < size; i++) {
if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
decodeOnlyPresentationTimestamps.remove(i);
return true;
}
}
return false;
}

@RequiresApi(23)
private void updateDrmSessionV23() throws ExoPlaybackException {
CryptoConfig cryptoConfig = sourceDrmSession.getCryptoConfig();
Expand Down
106 changes: 106 additions & 0 deletions ...y/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecRendererTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.oneByteSample;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.spy;
Expand Down Expand Up @@ -323,6 +326,90 @@ public void render_withReplaceStream_triggersOutputCallbacksInCorrectOrder() thr
inOrder.verify(renderer).onProcessedOutputBuffer(400);
}

@Test
public void render_afterEnableWithStartPositionUs_skipsSamplesBeforeStartPositionUs()
throws Exception {
Format format =
new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build();
FakeSampleStream fakeSampleStream =
createFakeSampleStream(format, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);

renderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {format},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 300,
/* offsetUs= */ 0);
renderer.start();
renderer.setCurrentStreamFinal();
long positionUs = 0;
while (!renderer.isEnded()) {
renderer.render(positionUs, SystemClock.elapsedRealtime());
positionUs += 100;
}

InOrder inOrder = inOrder(renderer);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 0, /* isDecodeOnly= */ true);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 100, /* isDecodeOnly= */ true);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 200, /* isDecodeOnly= */ true);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 300, /* isDecodeOnly= */ false);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 400, /* isDecodeOnly= */ false);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 500, /* isDecodeOnly= */ false);
}

@Test
public void render_afterPositionReset_skipsSamplesBeforeStartPositionUs() throws Exception {
Format format =
new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build();
FakeSampleStream fakeSampleStream =
createFakeSampleStream(format, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {format},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 400,
/* offsetUs= */ 0);
renderer.start();

renderer.resetPosition(/* positionUs= */ 200);
renderer.setCurrentStreamFinal();
long positionUs = 0;
while (!renderer.isEnded()) {
renderer.render(positionUs, SystemClock.elapsedRealtime());
positionUs += 100;
}

InOrder inOrder = inOrder(renderer);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 0, /* isDecodeOnly= */ true);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 100, /* isDecodeOnly= */ true);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 200, /* isDecodeOnly= */ false);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 300, /* isDecodeOnly= */ false);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 400, /* isDecodeOnly= */ false);
verifyProcessOutputBufferDecodeOnly(
inOrder, renderer, /* presentationTimeUs= */ 500, /* isDecodeOnly= */ false);
}

private FakeSampleStream createFakeSampleStream(Format format, long... sampleTimesUs) {
ImmutableList.Builder<FakeSampleStream.FakeSampleStreamItem> sampleListBuilder =
ImmutableList.builder();
Expand Down Expand Up @@ -429,4 +516,23 @@ protected DecoderReuseEvaluation canReuseCodec(
/* discardReasons= */ 0);
}
}

private static void verifyProcessOutputBufferDecodeOnly(
InOrder inOrder, MediaCodecRenderer renderer, long presentationTimeUs, boolean isDecodeOnly)
throws Exception {
inOrder
.verify(renderer)
.processOutputBuffer(
anyLong(),
anyLong(),
any(),
any(),
anyInt(),
anyInt(),
anyInt(),
eq(presentationTimeUs),
eq(isDecodeOnly),
anyBoolean(),
any());
}
}
2 changes: 1 addition & 1 deletion ...y/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEn
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* startPositionUs= */ 30_000,
/* offsetUs= */ 0);

mediaCodecVideoRenderer.start();
Expand Down

0 comments on commit cf02581

Please sign in to comment.