Skip to content

Commit bca46e6

Browse files
philncalvaris
philn
authored and
calvaris
committed
[GTK][WPE][WebRTC] webrtc/h264-baseline.html is crashing since added in r265005
https://bugs.webkit.org/show_bug.cgi?id=215005 Reviewed by Xabier Rodriguez-Calvar. The test was failing because the outgoing video track size updates were not properly applied to the capturer. Video size constraints are now properly propagated from the video capture source to the underlying GStreamerVideoCapturer. * LayoutTests/platform/glib/TestExpectations: * Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp: (WebCore::VideoFrameGStreamer::createFromPixelBuffer): * Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp: (WebCore::GStreamerCapturer::setupPipeline): * Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp: (WebCore::GStreamerVideoCaptureSource::settingsDidChange): (WebCore::GStreamerVideoCaptureSource::startProducingData): (WebCore::GStreamerVideoCaptureSource::setSizeFrameRateAndZoom): * Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h: * Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp: (WebCore::GStreamerVideoCapturer::setSize): (WebCore::GStreamerVideoCapturer::setFrameRate): * Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h: * Source/WebCore/platform/mediastream/gstreamer/MockDisplayCaptureSourceGStreamer.cpp: (WebCore::MockDisplayCaptureSourceGStreamer::create): * Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp: (WebCore::MockRealtimeVideoSource::create): (WebCore::MockRealtimeVideoSourceGStreamer::MockRealtimeVideoSourceGStreamer): (WebCore::MockRealtimeVideoSourceGStreamer::startProducingData): (WebCore::MockRealtimeVideoSourceGStreamer::updateSampleBuffer): (WebCore::MockRealtimeVideoSourceGStreamer::setSizeFrameRateAndZoom): * Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h: * Source/WebCore/platform/mock/MockRealtimeVideoSource.h: Canonical link: https://commits.webkit.org/274812@main
1 parent fe61765 commit bca46e6

11 files changed

Lines changed: 51 additions & 25 deletions

‎LayoutTests/platform/glib/TestExpectations‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2073,8 +2073,6 @@ webkit.org/b/235885 imported/w3c/web-platform-tests/webrtc/RTCRtpReceiver-getSyn
20732073
webkit.org/b/235885 imported/w3c/web-platform-tests/webrtc/protocol/rtp-clockrate.html [ Skip ]
20742074
webkit.org/b/235885 imported/w3c/web-platform-tests/webrtc-extensions/RTCRtpSynchronizationSource-senderCaptureTimeOffset.html [ Skip ]
20752075

2076-
# GStreamerRtpSenderBackend::setParameters() unimplemented.
2077-
webkit.org/b/215005 webkit.org/b/218787 webrtc/h264-baseline.html [ Failure Timeout ]
20782076
webkit.org/b/215007 webrtc/h264-high.html [ Failure Timeout ]
20792077

20802078
# We don't support spatial encoding yet.

‎Source/WebCore/platform/graphics/gstreamer/VideoFrameGStreamer.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,11 +299,13 @@ RefPtr<VideoFrameGStreamer> VideoFrameGStreamer::createFromPixelBuffer(Ref<Pixel
299299
}
300300

301301
auto outputBuffer = gst_sample_get_buffer(sample.get());
302+
gst_buffer_add_video_meta(outputBuffer, GST_VIDEO_FRAME_FLAG_NONE, format, width, height);
302303
if (metadata)
303304
webkitGstBufferSetVideoFrameTimeMetadata(outputBuffer, *metadata);
304305

305306
setBufferFields(outputBuffer, presentationTime, frameRate);
306307
} else {
308+
gst_buffer_add_video_meta(buffer.get(), GST_VIDEO_FRAME_FLAG_NONE, format, width, height);
307309
if (metadata)
308310
buffer = webkitGstBufferSetVideoFrameTimeMetadata(buffer.get(), *metadata);
309311

‎Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,8 @@ void GStreamerCapturer::setupPipeline()
197197
m_capsfilter = makeElement("capsfilter");
198198
m_sink = makeElement("appsink");
199199

200+
gst_util_set_object_arg(G_OBJECT(m_capsfilter.get()), "caps-change-mode", "delayed");
201+
200202
gst_app_sink_set_emit_signals(GST_APP_SINK(m_sink.get()), TRUE);
201203
g_object_set(m_sink.get(), "enable-last-sample", FALSE, nullptr);
202204
g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr);

‎Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ void GStreamerVideoCaptureSource::settingsDidChange(OptionSet<RealtimeMediaSourc
153153
if (m_deviceType == CaptureDevice::DeviceType::Window || m_deviceType == CaptureDevice::DeviceType::Screen)
154154
ensureIntrinsicSizeMaintainsAspectRatio();
155155

156-
if (m_capturer->setSize(size().width(), size().height()))
156+
if (m_capturer->setSize(size()))
157157
reconfigure = true;
158158
}
159159

@@ -186,7 +186,7 @@ void GStreamerVideoCaptureSource::startProducingData()
186186
m_capturer->setupPipeline();
187187

188188
if (m_deviceType == CaptureDevice::DeviceType::Camera)
189-
m_capturer->setSize(size().width(), size().height());
189+
m_capturer->setSize(size());
190190

191191
m_capturer->setFrameRate(frameRate());
192192
m_capturer->reconfigure();
@@ -304,6 +304,15 @@ void GStreamerVideoCaptureSource::generatePresets()
304304
setSupportedPresets(WTFMove(presets));
305305
}
306306

307+
void GStreamerVideoCaptureSource::setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate, std::optional<double> zoom)
308+
{
309+
RealtimeVideoCaptureSource::setSizeFrameRateAndZoom(width, height, frameRate, zoom);
310+
if (!width || !height)
311+
return;
312+
313+
m_capturer->setSize({ *width, *height });
314+
}
315+
307316
#undef GST_CAT_DEFAULT
308317

309318
} // namespace WebCore

‎Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ class GStreamerVideoCaptureSource : public RealtimeVideoCaptureSource, GStreamer
5858
void stopProducingData() override;
5959
bool canResizeVideoFrames() const final { return true; }
6060
void generatePresets() override;
61-
61+
void setSizeFrameRateAndZoom(std::optional<int>, std::optional<int>, std::optional<double>, std::optional<double>) override;
6262

6363
mutable std::optional<RealtimeMediaSourceCapabilities> m_capabilities;
6464
mutable std::optional<RealtimeMediaSourceSettings> m_currentSettings;

‎Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -124,14 +124,16 @@ GstElement* GStreamerVideoCapturer::createConverter()
124124
return bin;
125125
}
126126

127-
bool GStreamerVideoCapturer::setSize(int width, int height)
127+
bool GStreamerVideoCapturer::setSize(const IntSize& size)
128128
{
129129
if (isCapturingDisplay()) {
130130
// Pipewiresrc doesn't seem to support caps re-negotiation and framerate configuration properly.
131131
GST_FIXME_OBJECT(m_pipeline.get(), "Resizing disabled on display capture source");
132132
return true;
133133
}
134134

135+
int width = size.width();
136+
int height = size.height();
135137
if (!width || !height)
136138
return false;
137139

@@ -141,13 +143,14 @@ bool GStreamerVideoCapturer::setSize(int width, int height)
141143
return true;
142144
}
143145

146+
if (UNLIKELY(!m_capsfilter))
147+
return false;
148+
144149
GST_INFO_OBJECT(m_pipeline.get(), "Setting size to %dx%d", width, height);
150+
m_size = size;
145151
m_caps = adoptGRef(gst_caps_copy(m_caps.get()));
146152
gst_caps_set_simple(m_caps.get(), "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, nullptr);
147153

148-
if (!m_capsfilter)
149-
return false;
150-
151154
g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr);
152155
return true;
153156
}
@@ -161,7 +164,6 @@ bool GStreamerVideoCapturer::setFrameRate(double frameRate)
161164
}
162165

163166
int numerator, denominator;
164-
165167
gst_util_double_to_fraction(frameRate, &numerator, &denominator);
166168

167169
if (numerator < -G_MAXINT) {
@@ -174,15 +176,14 @@ bool GStreamerVideoCapturer::setFrameRate(double frameRate)
174176
return false;
175177
}
176178

179+
if (UNLIKELY(!m_capsfilter))
180+
return false;
181+
177182
m_caps = adoptGRef(gst_caps_copy(m_caps.get()));
178183
gst_caps_set_simple(m_caps.get(), "framerate", GST_TYPE_FRACTION, numerator, denominator, nullptr);
179184

180-
if (!m_capsfilter)
181-
return false;
182-
183185
GST_INFO_OBJECT(m_pipeline.get(), "Setting framerate to %f fps", frameRate);
184186
g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr);
185-
186187
return true;
187188
}
188189

‎Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ class GStreamerVideoCapturer final : public GStreamerCapturer {
4848
void setSinkVideoFrameCallback(SinkVideoFrameCallback&&);
4949

5050
private:
51-
bool setSize(int width, int height);
51+
bool setSize(const IntSize&);
52+
const IntSize& size() const { return m_size; }
5253
bool setFrameRate(double);
5354
void reconfigure();
5455

@@ -60,6 +61,7 @@ class GStreamerVideoCapturer final : public GStreamerCapturer {
6061
std::optional<NodeAndFD> m_nodeAndFd;
6162
GRefPtr<GstElement> m_videoSrcMIMETypeFilter;
6263
std::pair<unsigned long, SinkVideoFrameCallback> m_sinkVideoFrameCallback;
64+
IntSize m_size;
6365
};
6466

6567
} // namespace WebCore

‎Source/WebCore/platform/mediastream/gstreamer/MockDisplayCaptureSourceGStreamer.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ namespace WebCore {
3030

3131
CaptureSourceOrError MockDisplayCaptureSourceGStreamer::create(const CaptureDevice& device, MediaDeviceHashSalts&& hashSalts, const MediaConstraints* constraints, PageIdentifier pageIdentifier)
3232
{
33-
auto mockSource = adoptRef(*new MockRealtimeVideoSourceGStreamer(String { device.persistentId() }, AtomString { device.label() }, MediaDeviceHashSalts { hashSalts }));
33+
auto mockSource = adoptRef(*new MockRealtimeVideoSourceGStreamer(String { device.persistentId() }, AtomString { device.label() }, MediaDeviceHashSalts { hashSalts }, pageIdentifier));
3434

3535
if (constraints) {
3636
if (auto error = mockSource->applyConstraints(*constraints))

‎Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.cpp‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434

3535
namespace WebCore {
3636

37-
CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts, const MediaConstraints* constraints, PageIdentifier)
37+
CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts, const MediaConstraints* constraints, PageIdentifier pageIdentifier)
3838
{
3939
#ifndef NDEBUG
4040
auto device = MockRealtimeMediaSourceCenter::mockDeviceWithPersistentID(deviceID);
@@ -43,7 +43,7 @@ CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomStri
4343
return CaptureSourceOrError({ "No mock camera device"_s , MediaAccessDenialReason::PermissionDenied });
4444
#endif
4545

46-
Ref<RealtimeMediaSource> source = adoptRef(*new MockRealtimeVideoSourceGStreamer(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts)));
46+
Ref<RealtimeMediaSource> source = adoptRef(*new MockRealtimeVideoSourceGStreamer(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts), pageIdentifier));
4747
if (constraints) {
4848
if (auto error = source->applyConstraints(*constraints))
4949
return CaptureSourceOrError({ WTFMove(error->badConstraint), MediaAccessDenialReason::InvalidConstraint });
@@ -52,8 +52,8 @@ CaptureSourceOrError MockRealtimeVideoSource::create(String&& deviceID, AtomStri
5252
return source;
5353
}
5454

55-
MockRealtimeVideoSourceGStreamer::MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts)
56-
: MockRealtimeVideoSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts), { })
55+
MockRealtimeVideoSourceGStreamer::MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&& hashSalts, PageIdentifier pageIdentifier)
56+
: MockRealtimeVideoSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalts), pageIdentifier)
5757
{
5858
ensureGStreamerInitialized();
5959
auto& singleton = GStreamerVideoCaptureDeviceManager::singleton();
@@ -86,7 +86,7 @@ MockRealtimeVideoSourceGStreamer::~MockRealtimeVideoSourceGStreamer()
8686
void MockRealtimeVideoSourceGStreamer::startProducingData()
8787
{
8888
if (deviceType() == CaptureDevice::DeviceType::Camera)
89-
m_capturer->setSize(size().width(), size().height());
89+
m_capturer->setSize(size());
9090

9191
m_capturer->setFrameRate(frameRate());
9292
m_capturer->start();
@@ -108,7 +108,7 @@ void MockRealtimeVideoSourceGStreamer::captureEnded()
108108

109109
void MockRealtimeVideoSourceGStreamer::updateSampleBuffer()
110110
{
111-
auto imageBuffer = this->imageBuffer();
111+
RefPtr imageBuffer = this->imageBuffer();
112112
if (!imageBuffer)
113113
return;
114114

@@ -119,7 +119,7 @@ void MockRealtimeVideoSourceGStreamer::updateSampleBuffer()
119119
VideoFrameTimeMetadata metadata;
120120
metadata.captureTime = MonotonicTime::now().secondsSinceEpoch();
121121
auto presentationTime = MediaTime::createWithDouble((elapsedTime()).seconds());
122-
auto videoFrame = VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D, videoFrameRotation(), presentationTime, size(), frameRate(), false, WTFMove(metadata));
122+
auto videoFrame = VideoFrameGStreamer::createFromPixelBuffer(pixelBuffer.releaseNonNull(), VideoFrameGStreamer::CanvasContentType::Canvas2D, videoFrameRotation(), presentationTime, m_capturer->size(), frameRate(), false, WTFMove(metadata));
123123
if (!videoFrame)
124124
return;
125125

@@ -128,6 +128,16 @@ void MockRealtimeVideoSourceGStreamer::updateSampleBuffer()
128128
gst_app_src_push_sample(GST_APP_SRC_CAST(m_capturer->source()), videoFrame->sample());
129129
}
130130

131+
void MockRealtimeVideoSourceGStreamer::setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double> frameRate, std::optional<double> zoom)
132+
{
133+
MockRealtimeVideoSource::setSizeFrameRateAndZoom(width, height, frameRate, zoom);
134+
135+
if (!width || !height)
136+
return;
137+
138+
m_capturer->setSize({ *width, *height });
139+
}
140+
131141
} // namespace WebCore
132142

133143
#endif // ENABLE(MEDIA_STREAM) && USE(GSTREAMER)

‎Source/WebCore/platform/mediastream/gstreamer/MockRealtimeVideoSourceGStreamer.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ namespace WebCore {
3131

3232
class MockRealtimeVideoSourceGStreamer final : public MockRealtimeVideoSource, GStreamerCapturer::Observer {
3333
public:
34-
MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&&);
34+
MockRealtimeVideoSourceGStreamer(String&& deviceID, AtomString&& name, MediaDeviceHashSalts&&, PageIdentifier);
3535
~MockRealtimeVideoSourceGStreamer();
3636

3737
// GStreamerCapturer::Observer
@@ -44,6 +44,8 @@ class MockRealtimeVideoSourceGStreamer final : public MockRealtimeVideoSource, G
4444
void stopProducingData() final;
4545
void updateSampleBuffer() final;
4646
bool canResizeVideoFrames() const final { return true; }
47+
void setSizeFrameRateAndZoom(std::optional<int> width, std::optional<int> height, std::optional<double>, std::optional<double>) final;
48+
4749
RefPtr<GStreamerVideoCapturer> m_capturer;
4850
};
4951

0 commit comments

Comments
 (0)