Start early-enabled renderers only after advancing the playing period

Renderers may be enabled for subsequent media items as soon as the current media item's renderer's isEnded() returns true. When a renderer is being enabled and the player is 'playing', that renderer is also started. When playing a mixed playlist of images and content with audio & video, the player may skip some image items because the early-starting of the audio renderer causes a clock update.

A solution is to only start the "early-enabled" renderers at the point of media transition and add a condition on DefaultMediaClock to use the standalone clock when reading-ahead and the renderer clock source is not in a started state.

Issue: #1017
PiperOrigin-RevId: 613231227
(cherry picked from commit 638b2a3)

RELEASENOTES.md

@@ -7,6 +7,9 @@
         is preloaded again.
     *   Apply the correct corresponding `TrackSelectionResult` to the playing
         period in track reselection.
+    *   Start early-enabled renderers only after advancing the playing period
+        when transitioning between media items
+        ([#1017](https://github.com/androidx/media/issues/1017)).
 *   Transformer:
     *   Add workaround for exception thrown due to `MediaMuxer` not supporting
         negative presentation timestamps before API 30.

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultMediaClock.java

@@ -15,6 +15,8 @@
  */
 package androidx.media3.exoplayer;
 
+import static androidx.media3.exoplayer.Renderer.STATE_STARTED;
+
 import androidx.annotation.Nullable;
 import androidx.media3.common.PlaybackException;
 import androidx.media3.common.PlaybackParameters;
@@ -192,12 +194,14 @@ private void syncClocks(boolean isReadingAhead) {
   }
 
   private boolean shouldUseStandaloneClock(boolean isReadingAhead) {
-    // Use the standalone clock if the clock providing renderer is not set or has ended. Also use
+    // Use the standalone clock if the clock providing renderer is not set or has ended. Use the
+    // standalone clock if reading ahead and the renderer is not in a started state. Also use
     // the standalone clock if the renderer is not ready and we have finished reading the stream or
     // are reading ahead to avoid getting stuck if tracks in the current period have uneven
     // durations. See: https://github.com/google/ExoPlayer/issues/1874.
     return rendererClockSource == null
         || rendererClockSource.isEnded()
+        || (isReadingAhead && rendererClockSource.getState() != STATE_STARTED)
         || (!rendererClockSource.isReady()
             && (isReadingAhead || rendererClockSource.hasReadStreamToEnd()));
   }

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java

@@ -18,6 +18,9 @@
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Util.castNonNull;
 import static androidx.media3.common.util.Util.msToUs;
+import static androidx.media3.exoplayer.Renderer.STATE_DISABLED;
+import static androidx.media3.exoplayer.Renderer.STATE_ENABLED;
+import static androidx.media3.exoplayer.Renderer.STATE_STARTED;
 import static androidx.media3.exoplayer.audio.AudioSink.OFFLOAD_MODE_DISABLED;
 import static java.lang.Math.max;
 import static java.lang.Math.min;
@@ -76,6 +79,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -1777,7 +1781,7 @@ private void maybeTriggerPendingMessages(long oldPeriodPositionUs, long newPerio
   }
 
   private void ensureStopped(Renderer renderer) {
-    if (renderer.getState() == Renderer.STATE_STARTED) {
+    if (renderer.getState() == STATE_STARTED) {
       renderer.stop();
     }
   }
@@ -2319,6 +2323,16 @@ private void maybeUpdatePlayingPeriod() throws ExoPlaybackException {
               Player.DISCONTINUITY_REASON_AUTO_TRANSITION);
       resetPendingPauseAtEndOfPeriod();
       updatePlaybackPositions();
+      if (playbackInfo.playbackState == Player.STATE_READY) {
+        for (int i = 0; i < renderers.length; i++) {
+          if (renderers[i].getState() == STATE_ENABLED
+              && queue.getPlayingPeriod() != null
+              && Objects.equals(
+                  renderers[i].getStream(), queue.getPlayingPeriod().sampleStreams[i])) {
+            renderers[i].start();
+          }
+        }
+      }
       allowRenderersToRenderStartOfStreams();
       advancedPlayingPeriod = true;
     }
@@ -2665,7 +2679,7 @@ private void enableRenderer(int rendererIndex, boolean wasRendererEnabled, long
       return;
     }
     MediaPeriodHolder periodHolder = queue.getReadingPeriod();
-    boolean mayRenderStartOfStream = periodHolder == queue.getPlayingPeriod();
+    boolean arePlayingAndReadingTheSamePeriod = periodHolder == queue.getPlayingPeriod();
     TrackSelectorResult trackSelectorResult = periodHolder.getTrackSelectorResult();
     RendererConfiguration rendererConfiguration =
         trackSelectorResult.rendererConfigurations[rendererIndex];
@@ -2684,7 +2698,7 @@ private void enableRenderer(int rendererIndex, boolean wasRendererEnabled, long
         periodHolder.sampleStreams[rendererIndex],
         rendererPositionUs,
         joining,
-        mayRenderStartOfStream,
+        /* mayRenderStartOfStream= */ arePlayingAndReadingTheSamePeriod,
         startPositionUs,
         periodHolder.getRendererOffset(),
         periodHolder.info.id);
@@ -2703,8 +2717,8 @@ public void onWakeup() {
         });
 
     mediaClock.onRendererEnabled(renderer);
-    // Start the renderer if playing.
-    if (playing) {
+    // Start the renderer if playing and the Playing and Reading periods are the same.
+    if (playing && arePlayingAndReadingTheSamePeriod) {
       renderer.start();
     }
   }
@@ -3224,7 +3238,7 @@ private static Format[] getFormats(ExoTrackSelection newSelection) {
   }
 
   private static boolean isRendererEnabled(Renderer renderer) {
-    return renderer.getState() != Renderer.STATE_DISABLED;
+    return renderer.getState() != STATE_DISABLED;
   }
 
   private static final class SeekPosition {

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/PlaylistPlaybackTest.java

@@ -15,6 +15,9 @@
  */
 package androidx.media3.exoplayer.e2etest;
 
+import static com.google.common.truth.Truth.assertThat;
+import static org.robolectric.annotation.GraphicsMode.Mode.NATIVE;
+
 import android.content.Context;
 import android.graphics.SurfaceTexture;
 import android.net.Uri;
@@ -23,6 +26,7 @@
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.Player;
+import androidx.media3.common.util.Clock;
 import androidx.media3.exoplayer.ExoPlayer;
 import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
 import androidx.media3.exoplayer.source.MediaSource;
@@ -38,9 +42,11 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.robolectric.annotation.GraphicsMode;
 
 /** End-to-end tests for playlists. */
 @RunWith(AndroidJUnit4.class)
+@GraphicsMode(value = NATIVE)
 public final class PlaylistPlaybackTest {
 
   @Rule
@@ -141,4 +147,39 @@ public void test_subtitle() throws Exception {
     DumpFileAsserts.assertOutput(
         applicationContext, playbackOutput, "playbackdumps/playlists/playlist_with_subtitles.dump");
   }
+
+  @Test
+  public void testPlaylist_withImageAndAudioVideoItems_rendersExpectedContent() throws Exception {
+    Context applicationContext = ApplicationProvider.getApplicationContext();
+    CapturingRenderersFactory renderersFactory = new CapturingRenderersFactory(applicationContext);
+    Clock clock = new FakeClock(/* isAutoAdvancing= */ true);
+    ExoPlayer player =
+        new ExoPlayer.Builder(applicationContext, renderersFactory).setClock(clock).build();
+    PlaybackOutput playbackOutput = PlaybackOutput.register(player, renderersFactory);
+    long durationMs = 5 * C.MILLIS_PER_SECOND;
+    player.setMediaItems(
+        ImmutableList.of(
+            new MediaItem.Builder()
+                .setUri("asset:///media/png/media3test.png")
+                .setImageDurationMs(durationMs)
+                .build(),
+            new MediaItem.Builder()
+                .setUri("asset:///media/png/media3test.png")
+                .setImageDurationMs(durationMs)
+                .build(),
+            MediaItem.fromUri("asset:///media/mp4/sample.mp4")));
+    player.prepare();
+
+    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY);
+    long playerStartedMs = clock.elapsedRealtime();
+    player.play();
+    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
+    long playbackDurationMs = clock.elapsedRealtime() - playerStartedMs;
+    player.release();
+
+    // Playback duration should be greater than the sum of the image item durations.
+    assertThat(playbackDurationMs).isGreaterThan(durationMs * 2);
+    DumpFileAsserts.assertOutput(
+        applicationContext, playbackOutput, "playbackdumps/playlists/image_av_playlist.dump");
+  }
 }