Composition preview: play multiple images

PiperOrigin-RevId: 614690669

libraries/common/src/main/java/androidx/media3/common/util/ConstantRateTimestampIterator.java

@@ -33,7 +33,6 @@ public final class ConstantRateTimestampIterator implements TimestampIterator {
   private final long durationUs;
   private final float frameRate;
   private final double framesDurationUs;
-  private final long startingTimestampUs;
   private final int totalNumberOfFramesToAdd;
 
   private int framesAdded;
@@ -47,26 +46,10 @@ public final class ConstantRateTimestampIterator implements TimestampIterator {
   public ConstantRateTimestampIterator(
       @IntRange(from = 1) long durationUs,
       @FloatRange(from = 0, fromInclusive = false) float frameRate) {
-    this(durationUs, frameRate, /* startingTimestampUs= */ 0);
-  }
-
-  /**
-   * Creates an instance that outputs timestamps from {@code startingTimestampUs}.
-   *
-   * @param durationUs The duration the timestamps should span over, in microseconds.
-   * @param frameRate The frame rate in frames per second.
-   * @param startingTimestampUs The first timestamp output from the iterator.
-   */
-  public ConstantRateTimestampIterator(
-      @IntRange(from = 1) long durationUs,
-      @FloatRange(from = 0, fromInclusive = false) float frameRate,
-      @IntRange(from = 0) long startingTimestampUs) {
     checkArgument(durationUs > 0);
     checkArgument(frameRate > 0);
-    checkArgument(startingTimestampUs >= 0);
     this.durationUs = durationUs;
     this.frameRate = frameRate;
-    this.startingTimestampUs = startingTimestampUs;
     this.totalNumberOfFramesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
     framesDurationUs = C.MICROS_PER_SECOND / frameRate;
   }
@@ -84,7 +67,7 @@ public long next() {
 
   @Override
   public ConstantRateTimestampIterator copyOf() {
-    return new ConstantRateTimestampIterator(durationUs, frameRate, startingTimestampUs);
+    return new ConstantRateTimestampIterator(durationUs, frameRate);
   }
 
   @Override
@@ -97,7 +80,7 @@ public long getLastTimestampUs() {
 
   /** Returns the timestamp after {@code numberOfFrames}, in microseconds. */
   private long getTimestampUsAfter(int numberOfFrames) {
-    long timestampUs = round(startingTimestampUs + framesDurationUs * numberOfFrames);
+    long timestampUs = round(framesDurationUs * numberOfFrames);
     // Check for possible overflow.
     checkState(timestampUs >= 0);
     return timestampUs;

libraries/common/src/test/java/androidx/media3/common/util/ConstantRateTimestampIteratorTest.java

@@ -77,30 +77,6 @@ public void timestampIterator_smallDuration_generatesEmptyIterator() throws Exce
     assertThat(constantRateTimestampIterator.getLastTimestampUs()).isEqualTo(C.TIME_UNSET);
   }
 
-  @Test
-  public void timestampIterator_withNonZeroStartingTime_firstOutputsStartingTimestamp() {
-    ConstantRateTimestampIterator constantRateTimestampIterator =
-        new ConstantRateTimestampIterator(
-            /* durationUs= */ C.MICROS_PER_SECOND,
-            /* frameRate= */ 2,
-            /* startingTimestampUs= */ 1234);
-
-    assertThat(constantRateTimestampIterator.next()).isEqualTo(1234);
-    assertThat(constantRateTimestampIterator.getLastTimestampUs())
-        .isEqualTo(1234 + C.MICROS_PER_SECOND / 2);
-  }
-
-  @Test
-  public void copyOf_withNonZeroStartingTime_firstOutputsStartingTimestamp() {
-    ConstantRateTimestampIterator constantRateTimestampIterator =
-        new ConstantRateTimestampIterator(
-            /* durationUs= */ C.MICROS_PER_SECOND,
-            /* frameRate= */ 2,
-            /* startingTimestampUs= */ 1234);
-
-    assertThat(constantRateTimestampIterator.copyOf().next()).isEqualTo(1234);
-  }
-
   private static List<Long> generateList(TimestampIterator iterator) {
     ArrayList<Long> list = new ArrayList<>();
 

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java

@@ -705,12 +705,7 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame
       // the duration of the first video. Thus this correction is need to correct for the different
       // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
       long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs;
-      if (pendingInputStreamOffsetChange) {
-        compositingVideoSinkProvider.onStreamOffsetChange(
-            /* bufferPresentationTimeUs= */ bufferPresentationTimeUs,
-            /* streamOffsetUs= */ inputStreamOffsetUs);
-        pendingInputStreamOffsetChange = false;
-      }
+      maybeSetStreamOffsetChange(bufferPresentationTimeUs);
       lastBufferPresentationTimeUs = bufferPresentationTimeUs;
       if (isLastFrame) {
         finalBufferPresentationTimeUs = bufferPresentationTimeUs;
@@ -720,11 +715,29 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame
 
     @Override
     public boolean queueBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator) {
-      if (checkStateNotNull(videoFrameProcessor).queueInputBitmap(inputBitmap, timestampIterator)) {
-        lastBufferPresentationTimeUs = timestampIterator.getLastTimestampUs();
-        return true;
+      if (!maybeRegisterPendingInputStream()) {
+        return false;
       }
-      return false;
+
+      // The sink takes bitmaps with monotonically increasing, non-offset frame timestamps. Ensure
+      // the produced timestamps include the stream offset.
+      OffsetTimestampIterator offsetTimestampIterator =
+          new OffsetTimestampIterator(timestampIterator, inputStreamOffsetUs);
+      if (!checkStateNotNull(videoFrameProcessor)
+          .queueInputBitmap(inputBitmap, offsetTimestampIterator)) {
+        return false;
+      }
+
+      // Create a copy of iterator because we need to take the next timestamp but we must not alter
+      // the state of the iterator.
+      TimestampIterator copyTimestampIterator = offsetTimestampIterator.copyOf();
+      long bufferPresentationTimeUs = copyTimestampIterator.next();
+      long lastBufferPresentationTimeUs = copyTimestampIterator.getLastTimestampUs();
+      checkState(lastBufferPresentationTimeUs != C.TIME_UNSET);
+      maybeSetStreamOffsetChange(bufferPresentationTimeUs);
+      this.lastBufferPresentationTimeUs = lastBufferPresentationTimeUs;
+      finalBufferPresentationTimeUs = lastBufferPresentationTimeUs;
+      return true;
     }
 
     @Override
@@ -765,6 +778,35 @@ public void setStreamOffsetUs(long streamOffsetUs) {
       inputStreamOffsetUs = streamOffsetUs;
     }
 
+    private void maybeSetStreamOffsetChange(long bufferPresentationTimeUs) {
+      if (pendingInputStreamOffsetChange) {
+        compositingVideoSinkProvider.onStreamOffsetChange(
+            /* bufferPresentationTimeUs= */ bufferPresentationTimeUs,
+            /* streamOffsetUs= */ inputStreamOffsetUs);
+        pendingInputStreamOffsetChange = false;
+      }
+    }
+
+    /**
+     * Attempt to register any pending input stream to the video graph input and returns {@code
+     * true} if a pending stream was registered and/or there is no pending input stream waiting for
+     * registration, hence it's safe to queue images or frames to the video graph input.
+     */
+    private boolean maybeRegisterPendingInputStream() {
+      if (pendingInputStreamBufferPresentationTimeUs == C.TIME_UNSET) {
+        return true;
+      }
+      // An input stream is fully decoded, wait until all of its frames are released before queueing
+      // input frame from the next input stream.
+      if (compositingVideoSinkProvider.hasReleasedFrame(
+          pendingInputStreamBufferPresentationTimeUs)) {
+        maybeRegisterInputStream();
+        pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET;
+        return true;
+      }
+      return false;
+    }
+
     private void maybeRegisterInputStream() {
       if (inputFormat == null) {
         return;
@@ -785,6 +827,7 @@ private void maybeRegisterInputStream() {
                   inputFormat.height)
               .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
               .build());
+      finalBufferPresentationTimeUs = C.TIME_UNSET;
     }
 
     // CompositingVideoSinkProvider.Listener implementation
@@ -957,4 +1000,43 @@ public VideoFrameProcessor create(
               listener);
     }
   }
+
+  /**
+   * A {@link TimestampIterator} that wraps another {@link TimestampIterator} and adds an offset to
+   * the returnd timestamps.
+   */
+  private static class OffsetTimestampIterator implements TimestampIterator {
+
+    private final TimestampIterator timestampIterator;
+    private final long offset;
+
+    public OffsetTimestampIterator(TimestampIterator timestampIterator, long offset) {
+      this.timestampIterator = timestampIterator;
+      this.offset = offset;
+    }
+
+    @Override
+    public boolean hasNext() {
+      return timestampIterator.hasNext();
+    }
+
+    @Override
+    public long next() {
+      return offset + timestampIterator.next();
+    }
+
+    @Override
+    public long getLastTimestampUs() {
+      long last = timestampIterator.getLastTimestampUs();
+      if (last != C.TIME_UNSET) {
+        last += offset;
+      }
+      return last;
+    }
+
+    @Override
+    public TimestampIterator copyOf() {
+      return new OffsetTimestampIterator(timestampIterator.copyOf(), offset);
+    }
+  }
 }