Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: cherry-pick 7196a42b42ce from chromium (#36458)
* chore: cherry-pick 7196a42b42ce from chromium * chore: update patches Co-authored-by: PatchUp <73610968+patchup[bot]@users.noreply.github.com>
- Loading branch information
1 parent
1601265
commit 6338350
Showing
2 changed files
with
252 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,251 @@ | ||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 | ||
From: Johannes Kron <kron@chromium.org> | ||
Date: Thu, 17 Nov 2022 19:54:33 +0000 | ||
Subject: Reconfigure stream on window resize when using ScreenCaptureKit | ||
|
||
Bug: chromium:1352405 | ||
Change-Id: I315104eb4ee985a2d04ea90f01341129c6307501 | ||
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4027202 | ||
Reviewed-by: ccameron chromium <ccameron@chromium.org> | ||
Reviewed-by: Robert Sesek <rsesek@chromium.org> | ||
Commit-Queue: Johannes Kron <kron@chromium.org> | ||
Cr-Commit-Position: refs/heads/main@{#1072984} | ||
|
||
diff --git a/content/browser/media/capture/screen_capture_kit_device_mac.mm b/content/browser/media/capture/screen_capture_kit_device_mac.mm | ||
index c17f6f076fc852df85d5deb919434458b886db37..5d640a372f1649e0e2cb7e1828dbb3c66eb1e4d7 100644 | ||
--- a/content/browser/media/capture/screen_capture_kit_device_mac.mm | ||
+++ b/content/browser/media/capture/screen_capture_kit_device_mac.mm | ||
@@ -6,15 +6,18 @@ | ||
|
||
#import <ScreenCaptureKit/ScreenCaptureKit.h> | ||
|
||
+#include "base/mac/foundation_util.h" | ||
#include "base/mac/scoped_nsobject.h" | ||
#include "base/task/bind_post_task.h" | ||
#include "base/threading/thread_checker.h" | ||
#include "base/timer/timer.h" | ||
#include "content/browser/media/capture/io_surface_capture_device_base_mac.h" | ||
+#include "third_party/abseil-cpp/absl/types/optional.h" | ||
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_types.h" | ||
#include "ui/gfx/native_widget_types.h" | ||
|
||
-using SampleCallback = base::RepeatingCallback<void(gfx::ScopedInUseIOSurface)>; | ||
+using SampleCallback = base::RepeatingCallback<void(gfx::ScopedInUseIOSurface, | ||
+ absl::optional<gfx::Size>)>; | ||
using ErrorCallback = base::RepeatingClosure; | ||
|
||
API_AVAILABLE(macos(12.3)) | ||
@@ -45,17 +48,64 @@ - (void)stream:(SCStream*)stream | ||
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); | ||
if (!pixelBuffer) | ||
return; | ||
+ | ||
+ // Read out width, height and scaling from metadata to determine the captured | ||
+ // content size. | ||
+ absl::optional<gfx::Size> contentSize; | ||
+ CFArrayRef attachmentsArray = | ||
+ CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false); | ||
+ if (attachmentsArray && CFArrayGetCount(attachmentsArray) > 0) { | ||
+ CFDictionaryRef attachment = base::mac::CFCast<CFDictionaryRef>( | ||
+ CFArrayGetValueAtIndex(attachmentsArray, 0)); | ||
+ if (attachment) { | ||
+ CFDictionaryRef contentRectValue = base::mac::CFCast<CFDictionaryRef>( | ||
+ CFDictionaryGetValue(attachment, SCStreamFrameInfoContentRect)); | ||
+ CFNumberRef contentScaleValue = base::mac::CFCast<CFNumberRef>( | ||
+ CFDictionaryGetValue(attachment, SCStreamFrameInfoContentScale)); | ||
+ if (contentRectValue && contentScaleValue) { | ||
+ CGRect contentRect = {}; | ||
+ bool succeed = CGRectMakeWithDictionaryRepresentation(contentRectValue, | ||
+ &contentRect); | ||
+ float contentScale = 1.0f; | ||
+ succeed &= CFNumberGetValue(contentScaleValue, kCFNumberFloatType, | ||
+ &contentScale); | ||
+ if (succeed) { | ||
+ contentSize.emplace(round(contentRect.size.width / contentScale), | ||
+ round(contentRect.size.height / contentScale)); | ||
+ } | ||
+ } | ||
+ } | ||
+ } | ||
IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); | ||
if (!ioSurface) | ||
return; | ||
_sampleCallback.Run( | ||
- gfx::ScopedInUseIOSurface(ioSurface, base::scoped_policy::RETAIN)); | ||
+ gfx::ScopedInUseIOSurface(ioSurface, base::scoped_policy::RETAIN), | ||
+ contentSize); | ||
} | ||
|
||
- (void)stream:(SCStream*)stream didStopWithError:(NSError*)error { | ||
_errorCallback.Run(); | ||
} | ||
|
||
++ (base::scoped_nsobject<SCStreamConfiguration>) | ||
+ createStreamConfigurationWithFrameSize:(gfx::Size)frameSize | ||
+ destRectInFrame:(gfx::RectF)destRectInFrame | ||
+ frameRate:(float)frameRate { | ||
+ base::scoped_nsobject<SCStreamConfiguration> config( | ||
+ [[SCStreamConfiguration alloc] init]); | ||
+ [config setWidth:frameSize.width()]; | ||
+ [config setHeight:frameSize.height()]; | ||
+ [config setPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; | ||
+ [config setDestinationRect:destRectInFrame.ToCGRect()]; | ||
+ [config setBackgroundColor:CGColorGetConstantColor(kCGColorBlack)]; | ||
+ [config setScalesToFit:YES]; | ||
+ [config setShowsCursor:YES]; | ||
+ [config setColorSpaceName:kCGColorSpaceSRGB]; | ||
+ [config setMinimumFrameInterval:CMTimeMakeWithSeconds(1 / frameRate, 1)]; | ||
+ return config; | ||
+} | ||
+ | ||
@end | ||
|
||
namespace content { | ||
@@ -95,7 +145,6 @@ void OnShareableContentCreated( | ||
} | ||
|
||
base::scoped_nsobject<SCContentFilter> filter; | ||
- gfx::Size content_size; | ||
switch (source_.type) { | ||
case DesktopMediaID::TYPE_SCREEN: | ||
for (SCDisplay* display : [content displays]) { | ||
@@ -112,7 +161,8 @@ void OnShareableContentCreated( | ||
filter.reset([[SCContentFilter alloc] | ||
initWithDisplay:display | ||
excludingWindows:exclude_windows]); | ||
- content_size = gfx::Size([display width], [display height]); | ||
+ stream_config_content_size_ = | ||
+ gfx::Size([display width], [display height]); | ||
break; | ||
} | ||
} | ||
@@ -123,7 +173,7 @@ void OnShareableContentCreated( | ||
filter.reset([[SCContentFilter alloc] | ||
initWithDesktopIndependentWindow:window]); | ||
CGRect frame = [window frame]; | ||
- content_size = gfx::Size(frame.size); | ||
+ stream_config_content_size_ = gfx::Size(frame.size); | ||
break; | ||
} | ||
} | ||
@@ -142,22 +192,16 @@ void OnShareableContentCreated( | ||
gfx::RectF dest_rect_in_frame; | ||
actual_capture_format_ = capture_params().requested_format; | ||
actual_capture_format_.pixel_format = media::PIXEL_FORMAT_NV12; | ||
- ComputeFrameSizeAndDestRect(content_size, actual_capture_format_.frame_size, | ||
+ ComputeFrameSizeAndDestRect(stream_config_content_size_, | ||
+ actual_capture_format_.frame_size, | ||
dest_rect_in_frame); | ||
- | ||
- base::scoped_nsobject<SCStreamConfiguration> config( | ||
- [[SCStreamConfiguration alloc] init]); | ||
- [config setWidth:actual_capture_format_.frame_size.width()]; | ||
- [config setHeight:actual_capture_format_.frame_size.height()]; | ||
- [config setPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; | ||
- [config setDestinationRect:dest_rect_in_frame.ToCGRect()]; | ||
- [config setBackgroundColor:CGColorGetConstantColor(kCGColorBlack)]; | ||
- [config setScalesToFit:YES]; | ||
- [config setShowsCursor:YES]; | ||
- [config setColorSpaceName:kCGColorSpaceSRGB]; | ||
- [config | ||
- setMinimumFrameInterval:CMTimeMakeWithSeconds( | ||
- 1 / actual_capture_format_.frame_rate, 1)]; | ||
+ base::scoped_nsobject<SCStreamConfiguration> config = | ||
+ [ScreenCaptureKitDeviceHelper | ||
+ createStreamConfigurationWithFrameSize:actual_capture_format_ | ||
+ .frame_size | ||
+ destRectInFrame:dest_rect_in_frame | ||
+ frameRate:actual_capture_format_ | ||
+ .frame_rate]; | ||
stream_.reset([[SCStream alloc] initWithFilter:filter | ||
configuration:config | ||
delegate:helper_]); | ||
@@ -203,9 +247,71 @@ void OnStreamStopped(bool error) { | ||
return; | ||
} | ||
} | ||
- void OnStreamSample(gfx::ScopedInUseIOSurface io_surface) { | ||
- // TODO(https://crbug.com/1309653): Reconfigure the stream if the IOSurface | ||
- // should be resized. | ||
+ void OnStreamSample(gfx::ScopedInUseIOSurface io_surface, | ||
+ absl::optional<gfx::Size> content_size) { | ||
+ if (requested_capture_format_) { | ||
+ // Does the size of io_surface match the requested format? | ||
+ size_t io_surface_width = IOSurfaceGetWidth(io_surface); | ||
+ size_t io_surface_height = IOSurfaceGetHeight(io_surface); | ||
+ DVLOG(3) << "Waiting for new capture format, " | ||
+ << requested_capture_format_->frame_size.width() << " x " | ||
+ << requested_capture_format_->frame_size.height() | ||
+ << ". IO surface size " << io_surface_width << " x " | ||
+ << io_surface_height; | ||
+ if (static_cast<size_t>(requested_capture_format_->frame_size.width()) == | ||
+ io_surface_width && | ||
+ static_cast<size_t>(requested_capture_format_->frame_size.height()) == | ||
+ io_surface_height) { | ||
+ actual_capture_format_ = requested_capture_format_.value(); | ||
+ requested_capture_format_.reset(); | ||
+ } | ||
+ } else { | ||
+ // No current request for new capture format. Check to see if content_size | ||
+ // has changed and requires an updated configuration. | ||
+ if (content_size && | ||
+ (stream_config_content_size_.width() != content_size->width() || | ||
+ stream_config_content_size_.height() != content_size->height())) { | ||
+ DVLOG(3) << "Content size changed to " << content_size->width() << " x " | ||
+ << content_size->height() << ". It was " | ||
+ << stream_config_content_size_.width() << " x " | ||
+ << stream_config_content_size_.height(); | ||
+ stream_config_content_size_ = content_size.value(); | ||
+ gfx::RectF dest_rect_in_frame; | ||
+ gfx::Size new_frame_size; | ||
+ ComputeFrameSizeAndDestRect(stream_config_content_size_, new_frame_size, | ||
+ dest_rect_in_frame); | ||
+ if (new_frame_size.width() != | ||
+ actual_capture_format_.frame_size.width() || | ||
+ new_frame_size.height() != | ||
+ actual_capture_format_.frame_size.height()) { | ||
+ DVLOG(3) << "Calling updateConfiguration with new frame size: " | ||
+ << new_frame_size.width() << " x " | ||
+ << new_frame_size.height(); | ||
+ requested_capture_format_ = actual_capture_format_; | ||
+ requested_capture_format_->frame_size = new_frame_size; | ||
+ // Update stream configuration. | ||
+ base::scoped_nsobject<SCStreamConfiguration> config = | ||
+ [ScreenCaptureKitDeviceHelper | ||
+ createStreamConfigurationWithFrameSize: | ||
+ requested_capture_format_->frame_size | ||
+ destRectInFrame:dest_rect_in_frame | ||
+ frameRate: | ||
+ requested_capture_format_-> | ||
+ frame_rate]; | ||
+ [stream_ | ||
+ updateConfiguration:config | ||
+ completionHandler:^(NSError* _Nullable error) { | ||
+ if (error) { | ||
+ client()->OnError( | ||
+ media::VideoCaptureError::kScreenCaptureKitStreamError, | ||
+ FROM_HERE, "Error on updateConfiguration"); | ||
+ } | ||
+ }]; | ||
+ } | ||
+ } | ||
+ } | ||
+ // TODO(https://crbug.com/1352405): Set visible rect to make it possible to | ||
+ // crop the frame when it's rendered/encoded. | ||
OnReceivedIOSurfaceFromStream(io_surface, actual_capture_format_); | ||
} | ||
void OnStreamError() { | ||
@@ -257,6 +363,13 @@ void OnStop() override { | ||
// The actual format of the video frames that are sent to `client`. | ||
media::VideoCaptureFormat actual_capture_format_; | ||
|
||
+ // The requested format if a request to update the configuration has been | ||
+ // sent. | ||
+ absl::optional<media::VideoCaptureFormat> requested_capture_format_; | ||
+ | ||
+ // The size of the content at the time that we configured the stream. | ||
+ gfx::Size stream_config_content_size_; | ||
+ | ||
// Helper class that acts as output and delegate for `stream_`. | ||
base::scoped_nsobject<ScreenCaptureKitDeviceHelper> helper_; | ||
|