Skip to content

Commit ae6cd18

Browse files
youennfyouennf
committed
Add support for ImageCapture.grabFrame
rdar://148425176 https://bugs.webkit.org/show_bug.cgi?id=290916 Reviewed by Eric Carlson. Add support of ImageCapture::grabFrame by doing the following: - Introduce ImageCaptureVideoFrameObserver as a VideoFrameObserver on the ImageCapture track. - Convert the VideoFrame into an ImageBitmap. We handle the rotation so that drawing the ImageBitmap is similar to rendering the track via a media element. - The conversion requires conversion to RGB which is done in GPUProcess. For that purpose, we introduce a new MediaStrategy which uses an existing IPC call. Marking as failing in glib since there is a lack of mock rotation support. Covered by added test. * LayoutTests/fast/mediastream/image-capture-grabFrame-expected.txt: Added. * LayoutTests/fast/mediastream/image-capture-grabFrame.html: Added. * LayoutTests/platform/glib/TestExpectations: * Source/WebCore/Modules/mediastream/ImageCapture.cpp: (WebCore::ImageCapture::~ImageCapture): (WebCore::videoFrameOrientation): (WebCore::createImageBitmapViaDrawing): (WebCore::createImageBitmapFromNativeImage): (WebCore::createImageBitmap): (WebCore::ImageCaptureVideoFrameObserver::create): (WebCore::ImageCaptureVideoFrameObserver::~ImageCaptureVideoFrameObserver): (WebCore::ImageCaptureVideoFrameObserver::add): (WebCore::ImageCaptureVideoFrameObserver::stop): (WebCore::ImageCaptureVideoFrameObserver::ImageCaptureVideoFrameObserver): (WebCore::ImageCaptureVideoFrameObserver::processVideoFrame): (WebCore::ImageCapture::grabFrame): (WebCore::ImageCapture::stop): (WebCore::ImageCapture::stopGrabFrameObserver): * Source/WebCore/Modules/mediastream/ImageCapture.h: * Source/WebCore/Modules/mediastream/ImageCapture.idl: * Source/WebCore/platform/MediaStrategy.h: (WebCore::MediaStrategy::nativeImageFromVideoFrame): * Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp: (WebCore::MockRealtimeVideoSource::orientationChanged): * Source/WebKit/WebProcess/GPU/media/WebMediaStrategy.cpp: (WebKit::WebMediaStrategy::nativeImageFromVideoFrame): * Source/WebKit/WebProcess/GPU/media/WebMediaStrategy.h: Canonical link: https://commits.webkit.org/293243@main
1 parent e7b2eb3 commit ae6cd18

File tree

10 files changed

+415
-2
lines changed

10 files changed

+415
-2
lines changed

‎LayoutTests/fast/mediastream/image-capture-grabFrame-expected.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
2+
3+
PASS 'grabFrame()' on an 'ended' track should reject "InvalidStateError"
4+
PASS "grabFrame" should reject if the track ends before the 'grabFrame()' promise resolves
5+
PASS The image returned by 'grabFrame()' should have the same size as track settings
6+
PASS Validate 0 rotated frame
7+
PASS Validate 90 rotated frame
8+
PASS Validate 180 rotated frame
9+
PASS Validate 270 rotated frame
10+

‎LayoutTests/fast/mediastream/image-capture-grabFrame.html‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<meta charset='utf-8'>
5+
<title>ImageCapture grabFrame</title>
6+
<script src='../../resources/testharness.js'></script>
7+
<script src='../../resources/testharnessreport.js'></script>
8+
</head>
9+
<body>
10+
<video id=video autoplay playsinline></video>
11+
<br>
12+
<canvas id=canvas1></canvas>
13+
<canvas id=canvas2></canvas>
14+
<script>
15+
promise_test(async (test) => {
16+
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 640 } });
17+
const [track] = stream.getVideoTracks();
18+
test.add_cleanup(() => track.stop());
19+
20+
assert_equals(track.readyState, 'live');
21+
track.stop();
22+
assert_equals(track.readyState, 'ended');
23+
24+
const imageCapture = new ImageCapture(track);
25+
const promise = imageCapture.grabFrame();
26+
27+
let result;
28+
promise.then(
29+
(value) => { result = value; },
30+
(error) => { result = error; }
31+
);
32+
33+
await Promise.resolve();
34+
assert_equals(result['name'], 'InvalidStateError');
35+
return promise_rejects_dom(test, 'InvalidStateError', promise);
36+
37+
}, `'grabFrame()' on an 'ended' track should reject "InvalidStateError"`);
38+
39+
promise_test(async (test) => {
40+
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 640 } });
41+
const [track] = stream.getVideoTracks();
42+
test.add_cleanup(() => track.stop());
43+
44+
assert_equals(track.readyState, 'live');
45+
46+
const imageCapture = new ImageCapture(track);
47+
const promise = imageCapture.grabFrame();
48+
49+
track.stop();
50+
assert_equals(track.readyState, 'ended');
51+
52+
return promise_rejects_dom(test, 'OperationError', promise);
53+
54+
}, `"grabFrame" should reject if the track ends before the 'grabFrame()' promise resolves`);
55+
56+
promise_test(async (test) => {
57+
const stream = await navigator.mediaDevices.getUserMedia({ video: { width : 640 } });
58+
const [track] = stream.getVideoTracks();
59+
test.add_cleanup(() => track.stop());
60+
61+
const imageCapture = new ImageCapture(track);
62+
let settings = track.getSettings();
63+
assert_equals(settings.width, 640);
64+
65+
let image = await imageCapture.grabFrame();
66+
assert_equals(image.width, settings.width, "image width 1");
67+
assert_equals(image.height, settings.height, "image height 1");
68+
69+
await track.applyConstraints({ width: 320 });
70+
settings = track.getSettings();
71+
assert_equals(settings.width, 320);
72+
73+
image = await imageCapture.grabFrame();
74+
assert_equals(image.width, settings.width, "image width 2");
75+
assert_equals(image.height, settings.height, "image height 2");
76+
}, `The image returned by 'grabFrame()' should have the same size as track settings`);
77+
78+
async function validateImages(test, rotation)
79+
{
80+
if (!window.testRunner)
81+
return;
82+
testRunner.setMockCameraOrientation(rotation);
83+
84+
let width = 640;
85+
let height = 480;
86+
const stream = await navigator.mediaDevices.getUserMedia({ video: { width, height, frameRate : 5 } });
87+
const [track] = stream.getVideoTracks();
88+
test.add_cleanup(() => track.stop());
89+
90+
video.srcObject = stream;
91+
await video.play();
92+
93+
const imageCapture = new ImageCapture(track);
94+
const imagePromise = imageCapture.grabFrame();
95+
const videoFramePromise = new Promise(resolve => video.requestVideoFrameCallback(() => resolve(new VideoFrame(video))));
96+
97+
const image = await imagePromise;
98+
const videoFrame = await videoFramePromise;
99+
test.add_cleanup(() => videoFrame.close());
100+
101+
if (rotation === 90 || rotation === 270) {
102+
width = 480;
103+
height = 640;
104+
}
105+
106+
canvas1.width = width;
107+
canvas1.height = height;
108+
canvas1.getContext('2d').drawImage(image, 0, 0, width, height);
109+
const data1 = canvas1.getContext('2d').getImageData(0, 0, width, height).data;
110+
111+
canvas2.width = width;
112+
canvas2.height = height;
113+
if (rotation === 0) {
114+
canvas2.getContext('2d').reset();
115+
canvas2.getContext('2d').drawImage(videoFrame, 0, 0, 640, 480);
116+
} else if (rotation === 90) {
117+
canvas2.getContext('2d').reset();
118+
canvas2.getContext('2d').translate(240, 320);
119+
canvas2.getContext('2d').rotate(Math.PI / 2);
120+
canvas2.getContext('2d').drawImage(videoFrame, -320, -240, 640, 480);
121+
} else if (rotation === 180) {
122+
canvas2.getContext('2d').reset();
123+
canvas2.getContext('2d').translate(320, 240);
124+
canvas2.getContext('2d').rotate(-Math.PI);
125+
canvas2.getContext('2d').drawImage(videoFrame, -320, -240, 640, 480);
126+
} else if (rotation === 270) {
127+
canvas2.getContext('2d').reset();
128+
canvas2.getContext('2d').translate(240, 320);
129+
canvas2.getContext('2d').rotate(-Math.PI / 2);
130+
canvas2.getContext('2d').drawImage(videoFrame, -320, -240, 640, 480);
131+
}
132+
133+
const data2 = canvas2.getContext('2d').getImageData(0, 0, width, height).data;
134+
135+
let error = 10;
136+
let checkRoughlyEqual = (a, b) => {
137+
return Math.abs(a - b) < error;
138+
};
139+
140+
let differenceCounter = 0;
141+
for (let i = 0; i < data1.length; i = i + 4) {
142+
if (!checkRoughlyEqual(data1[i], data2[i]) || !checkRoughlyEqual(data1[i + 1], data2[i + 1]) || !checkRoughlyEqual(data1[i + 2], data2[i + 2]))
143+
differenceCounter++;
144+
}
145+
146+
const threshold = canvas1.width * canvas1.height / 25;
147+
assert_less_than(differenceCounter, threshold);
148+
}
149+
150+
promise_test(async (test) => {
151+
return validateImages(test, 0);
152+
}, "Validate 0 rotated frame");
153+
154+
promise_test(async (test) => {
155+
return validateImages(test, 90);
156+
}, "Validate 90 rotated frame");
157+
158+
promise_test(async (test) => {
159+
return validateImages(test, 180);
160+
}, "Validate 180 rotated frame");
161+
162+
promise_test(async (test) => {
163+
return validateImages(test, 270);
164+
}, "Validate 270 rotated frame");
165+
</script>
166+
</body>
167+
</html>

‎LayoutTests/platform/glib/TestExpectations‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2235,6 +2235,7 @@ webrtc/video-rotation-no-cvo.html [ Failure Timeout ]
22352235
webrtc/video-rotation-black.html [ Crash Failure Pass Timeout ]
22362236

22372237
fast/mediastream/video-rotation-clone.html [ Skip ]
2238+
fast/mediastream/image-capture-grabFrame.html [ Failure ]
22382239

22392240
# No AudioSession category handling in WPE/GTK ports.
22402241
fast/mediastream/microphone-interruption-and-audio-session.html [ Skip ]

0 commit comments

Comments
 (0)