diff options
author | Lorn Potter <lorn.potter@gmail.com> | 2022-10-24 10:20:26 +1000 |
---|---|---|
committer | Lorn Potter <lorn.potter@gmail.com> | 2023-02-08 13:40:47 +1000 |
commit | e98bce0c5ff5afa609fdd8cde65a0405a77a4fa8 (patch) | |
tree | 17e167260b7e90b19269b5d07ece45e2f082cde3 /src/plugins | |
parent | a128d5ecf40d8d670d4d3aa8b9ddc860baa3cb5e (diff) |
wasm: add camera input
Also add image capture and video recorder
Fixes: QTBUG-108459
Pick-to: 6.5
Change-Id: I8c036142802c03cc19d968f8fc18e8a44a3640cf
Reviewed-by: MikoĊaj Boc <Mikolaj.Boc@qt.io>
Diffstat (limited to 'src/plugins')
15 files changed, 1595 insertions, 43 deletions
diff --git a/src/plugins/multimedia/wasm/CMakeLists.txt b/src/plugins/multimedia/wasm/CMakeLists.txt index 25903b0b1..7c012af2a 100644 --- a/src/plugins/multimedia/wasm/CMakeLists.txt +++ b/src/plugins/multimedia/wasm/CMakeLists.txt @@ -8,10 +8,14 @@ qt_internal_add_plugin(QWasmMediaPlugin common/qwasmvideooutput.cpp common/qwasmvideooutput_p.h common/qwasmaudiooutput.cpp common/qwasmaudiooutput_p.h common/qwasmaudioinput.cpp common/qwasmaudioinput_p.h - + mediacapture/qwasmmediacapturesession.cpp mediacapture/qwasmmediacapturesession_p.h + mediacapture/qwasmmediarecorder.cpp mediacapture/qwasmmediarecorder_p.h + mediacapture/qwasmcamera.cpp mediacapture/qwasmcamera_p.h + mediacapture/qwasmimagecapture.cpp mediacapture/qwasmimagecapture_p.h INCLUDE_DIRECTORIES common mediaplayer + mediacapture LIBRARIES Qt::MultimediaPrivate Qt::CorePrivate diff --git a/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp b/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp index 041d10254..3b0751f1a 100644 --- a/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp +++ b/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp @@ -60,15 +60,14 @@ void QWasmVideoOutput::setVideoMode(QWasmVideoOutput::WasmVideoMode mode) void QWasmVideoOutput::start() { - qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; - if (m_video.isUndefined() || m_video.isNull()) { // error emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); return; } - if (m_currentVideoMode == QWasmVideoOutput::VideoOutput) { + switch (m_currentVideoMode) { + case QWasmVideoOutput::VideoOutput: { emscripten::val sourceObj = m_video.call<emscripten::val>("getAttribute", emscripten::val("src")); @@ -77,23 +76,40 @@ void QWasmVideoOutput::start() m_video.call<void>("setAttribute", emscripten::val("src"), m_source.toStdString()); m_video.call<void>("load"); } - } else { + } break; + case QWasmVideoOutput::Camera: { emscripten::val stream = m_video["srcObject"]; - if (!stream.isNull() || !stream.isUndefined()) { // camera device - emscripten::val vTracks = stream.call<emscripten::val>("getVideoTracks"); - if (vTracks["count"].as<int>() > 1) { - emscripten::val vSettings = vTracks[0].call<emscripten::val>("getSettings"); - double fRate = vSettings["frameRate"].as<double>(); - int width = vSettings["width"].as<int>(); - int height = vSettings["height"].as<int>(); + if (stream.isNull() || stream.isUndefined()) { // camera device + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "ERROR"; + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + } else { + emscripten::val videoTracks = stream.call<emscripten::val>("getVideoTracks"); + if (videoTracks.isNull() || videoTracks.isUndefined()) { + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "videoTracks is null"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("video surface error")); + return; + } + if (videoTracks["length"].as<int>() == 0) { + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "videoTracks count is 0"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("video surface error")); + return; + } + emscripten::val videoSettings = videoTracks[0].call<emscripten::val>("getSettings"); + if (!videoSettings.isNull() || !videoSettings.isUndefined()) { + // double fRate = videoSettings["frameRate"].as<double>(); TODO + const int width = videoSettings["width"].as<int>(); + const int height = videoSettings["height"].as<int>(); qCDebug(qWasmMediaVideoOutput) - << "frame rate" << fRate << "width" << width << "height" << height; + << "width" << width << "height" << height; updateVideoElementGeometry(QRect(0, 0, width, height)); } } - } + } break; + }; m_shouldStop = false; m_toBePaused = false; @@ -109,7 +125,7 @@ void QWasmVideoOutput::stop() if (m_video.isUndefined() || m_video.isNull()) { // error - emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); return; } m_shouldStop = true; @@ -207,30 +223,99 @@ void QWasmVideoOutput::setSource(const QUrl &url) void QWasmVideoOutput::addSourceElement(const QString &urlString) { - qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; - if (m_video.isUndefined() || m_video.isNull()) { // error emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); return; } emscripten::val document = emscripten::val::global("document"); - m_videoElementSource = document.call<emscripten::val>("createElement", std::string("source")); - if (m_videoElementSource.isNull() || m_videoElementSource.isUndefined()) { - // error - emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); - return; - } if (!urlString.isEmpty()) - m_videoElementSource.set("src", m_source.toStdString()); - - m_video.call<void>("appendChild", m_videoElementSource); + m_video.set("src", m_source.toStdString()); if (!urlString.isEmpty()) m_video.call<void>("load"); } +void QWasmVideoOutput::addCameraSourceElement(const std::string &id) +{ + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + if (mediaDevices.isNull() || mediaDevices.isUndefined()) { + qWarning() << "No media devices found"; + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + + qstdweb::PromiseCallbacks getUserMediaCallback{ + .thenFunc = + [this](emscripten::val stream) { + qCDebug(qWasmMediaVideoOutput) << "getUserMediaSuccess"; + + m_video.set("srcObject", stream); + + emscripten::val videoTracksObject = stream.call<emscripten::val>("getVideoTracks"); + if (videoTracksObject.isNull() || videoTracksObject.isUndefined()) { + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + + if (videoTracksObject.call<emscripten::val>("length").as<int>() == 0) { + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks"); + if (tracks.isNull() || tracks.isUndefined()) + return; + if (tracks["length"].as<int>() == 0) + return; + emscripten::val videoTrack = tracks[0]; + if (videoTrack.isNull() || videoTrack.isUndefined()) { + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + + emscripten::val currentVideoCapabilities = videoTrack.call<emscripten::val>("getCapabilities"); + if (currentVideoCapabilities.isNull() || currentVideoCapabilities.isUndefined()) { + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + + emscripten::val videoSettings = videoTrack.call<emscripten::val>("getSettings"); + if (videoSettings.isNull() || videoSettings.isUndefined()) { + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + + // TODO double fRate = videoSettings["frameRate"].as<double>(); + // const int width = videoSettings["width"].as<int>(); + // const int height = videoSettings["height"].as<int>(); + + }, + .catchFunc = + [](emscripten::val error) { + qCDebug(qWasmMediaVideoOutput) + << "getUserMedia fail" + << QString::fromStdString(error["name"].as<std::string>()) + << QString::fromStdString(error["message"].as<std::string>()); + } + }; + + emscripten::val constraints = emscripten::val::object(); + + constraints.set("audio", m_hasAudio); + + emscripten::val videoContraints = emscripten::val::object(); + videoContraints.set("exact", id); + videoContraints.set("deviceId", id); + constraints.set("video", videoContraints); + + // we do it this way as this prompts user for mic/camera permissions + qstdweb::Promise::make(mediaDevices, QStringLiteral("getUserMedia"), + std::move(getUserMediaCallback), constraints); +} + void QWasmVideoOutput::setSource(QIODevice *stream) { Q_UNUSED(stream) @@ -323,15 +408,28 @@ bool QWasmVideoOutput::isVideoSeekable() void QWasmVideoOutput::createVideoElement(const std::string &id) { - qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << id; // Create <video> element and add it to the page body emscripten::val document = emscripten::val::global("document"); emscripten::val body = document["body"]; + emscripten::val oldVideo = document.call<emscripten::val>("getElementsByClassName", + (m_currentVideoMode == QWasmVideoOutput::Camera + ? std::string("Camera") + : std::string("Video"))); + + // we don't provide alternate tracks + // but need to remove stale track + if (oldVideo["length"].as<int>() > 0) + oldVideo[0].call<void>("remove"); + m_videoSurfaceId = id; m_video = document.call<emscripten::val>("createElement", std::string("video")); m_video.set("id", m_videoSurfaceId.c_str()); + m_video.call<void>("setAttribute", std::string("class"), + (m_currentVideoMode == QWasmVideoOutput::Camera ? std::string("Camera") + : std::string("Video"))); // if video m_video.set("preload", "metadata"); @@ -348,10 +446,7 @@ void QWasmVideoOutput::createVideoElement(const std::string &id) body.call<void>("appendChild", m_video); // Create/add video source - emscripten::val videoElementGeometry = - document.call<emscripten::val>("createElement", std::string("source")); - // videoElementGeometry.set("src", m_source.toStdString()); - m_video.call<void>("appendChild", videoElementGeometry); + m_video.set("src", m_source.toStdString()); // Set position:absolute, which makes it possible to position the video // element using x,y. coordinates, relative to its parent (the page's <body> @@ -467,7 +562,8 @@ void QWasmVideoOutput::doElementCallbacks() updateVideoElementGeometry( QRect(0, 0, m_video["videoWidth"].as<int>(), m_video["videoHeight"].as<int>())); - void addCameraSourceElement(const std::string &id); + emit sizeChange(m_video["videoWidth"].as<int>(), m_video["videoHeight"].as<int>()); + }; m_resizeChangeEvent.reset(new qstdweb::EventCallback(m_video, "resize", resizeCallback)); @@ -654,9 +750,6 @@ void QWasmVideoOutput::doElementCallbacks() emscripten::val window = emscripten::val::global("window"); window.call<void>("addEventListener", std::string("beforeunload"), emscripten::val::module_property("mbeforeUnload")); - - emscripten::val document = emscripten::val::global("document"); - m_videoElementSource = document.call<emscripten::val>("createElement", std::string("source")); } void QWasmVideoOutput::updateVideoElementGeometry(const QRect &windowGeometry) @@ -791,6 +884,57 @@ void QWasmVideoOutput::videoFrameTimerCallback() // about 60 fps } +emscripten::val QWasmVideoOutput::getDeviceCapabilities() +{ + emscripten::val stream = m_video["srcObject"]; + if (!stream.isUndefined() || !stream["getVideoTracks"].isUndefined()) { + emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks"); + if (!tracks.isUndefined()) { + if (tracks["length"].as<int>() == 0) + return emscripten::val::undefined(); + + emscripten::val track = tracks[0]; + if (!track.isUndefined()) { + emscripten::val trackCaps = emscripten::val::undefined(); + if (!track["getCapabilities"].isUndefined()) + trackCaps = track.call<emscripten::val>("getCapabilities"); + else // firefox does not support getCapabilities + trackCaps = track.call<emscripten::val>("getSettings"); + + if (!trackCaps.isUndefined()) + return trackCaps; + } + } + } else { + // camera not started track capabilities not available + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("capabilities not available")); + } + + return emscripten::val::undefined(); +} + +bool QWasmVideoOutput::setDeviceSetting(const std::string &key, emscripten::val value) +{ + emscripten::val stream = m_video["srcObject"]; + if (stream.isNull() || stream.isUndefined() + || stream["getVideoTracks"].isUndefined()) + return false; + + emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks"); + if (!tracks.isNull() || !tracks.isUndefined()) { + if (tracks["length"].as<int>() == 0) + return false; + + emscripten::val track = tracks[0]; + emscripten::val contraint = emscripten::val::object(); + contraint.set(std::move(key), value); + track.call<emscripten::val>("applyConstraints", contraint); + return true; + } + + return false; +} + QT_END_NAMESPACE #include "moc_qwasmvideooutput_p.cpp" diff --git a/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h b/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h index 3984ef3f7..66ae6fd2b 100644 --- a/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h +++ b/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h @@ -48,7 +48,7 @@ public: void pause(); void setSurface(QVideoSink *surface); - emscripten::val surfaceElement(); // ? + emscripten::val surfaceElement(); bool isReady() const; @@ -71,9 +71,16 @@ public: void doElementCallbacks(); void updateVideoElementGeometry(const QRect &windowGeometry); void addSourceElement(const QString &urlString); + void addCameraSourceElement(const std::string &id); void removeSourceElement(); void setVideoMode(QWasmVideoOutput::WasmVideoMode mode); + void setHasAudio(bool needsAudio) { m_hasAudio = needsAudio; } + + bool hasCapability(const QString &cap); + emscripten::val getDeviceCapabilities(); + bool setDeviceSetting(const std::string &key, emscripten::val value); + // mediacapturesession has the videosink QVideoSink *m_wasmSink = nullptr; @@ -92,6 +99,7 @@ private: void checkNetworkState(); void videoComputeFrame(void *context); void videoFrameTimerCallback(); + void getDeviceSettings(); emscripten::val m_video = emscripten::val::undefined(); emscripten::val m_videoElementSource = emscripten::val::undefined(); @@ -103,6 +111,7 @@ private: bool m_shouldStop = false; bool m_toBePaused = false; bool m_isSeeking = false; + bool m_hasAudio = false; emscripten::val m_offscreenContext = emscripten::val::undefined(); QSize m_pendingVideoSize; QWasmVideoOutput::WasmVideoMode m_currentVideoMode = QWasmVideoOutput::VideoOutput; diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmcamera.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera.cpp new file mode 100644 index 000000000..f346ef75b --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera.cpp @@ -0,0 +1,403 @@ +// Copyright (C) 2022 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmcamera_p.h" +#include "qmediadevices.h" +#include <qcameradevice.h> +#include "private/qabstractvideobuffer_p.h" +#include "private/qplatformvideosink_p.h" +#include <private/qmemoryvideobuffer_p.h> +#include <private/qvideotexturehelper_p.h> + +#include "qwasmmediacapturesession_p.h" +#include <common/qwasmvideooutput_p.h> + +#include <emscripten/val.h> +#include <emscripten/bind.h> +#include <emscripten/html5.h> +#include <QUuid> + +#include <private/qstdweb_p.h> + +Q_LOGGING_CATEGORY(qWasmCamera, "qt.multimedia.wasm.camera") + +QWasmCamera::QWasmCamera(QCamera *camera) + : QPlatformCamera(camera), m_cameraOutput(new QWasmVideoOutput) +{ +} + +QWasmCamera::~QWasmCamera() = default; + +bool QWasmCamera::isActive() const +{ + return m_cameraActive; +} + +void QWasmCamera::setActive(bool active) +{ + if (!m_CaptureSession) { + emit error(QCamera::CameraError, QStringLiteral("video surface error")); + return; + } + + m_cameraOutput->setSurface(m_CaptureSession->videoSink()); + + m_cameraActive = active; + + if (m_cameraActive) + m_cameraOutput->start(); + else + m_cameraOutput->pause(); + + updateCameraFeatures(); + emit activeChanged(active); +} + +void QWasmCamera::setCamera(const QCameraDevice &camera) +{ + m_cameraOutput->setVideoMode(QWasmVideoOutput::Camera); + + constexpr QSize initialSize(0, 0); + constexpr QRect initialRect(QPoint(0, 0), initialSize); + m_cameraOutput->createVideoElement(camera.id().toStdString()); // videoElementId + m_cameraOutput->createOffscreenElement(initialSize); + m_cameraOutput->updateVideoElementGeometry(initialRect); + + const auto cameras = QMediaDevices::videoInputs(); + if (std::find(cameras.begin(), cameras.end(), camera) != cameras.end()) { + m_cameraDev = camera; + createCamera(m_cameraDev); + return; + } + + if (cameras.count() > 0) { + m_cameraDev = camera; + createCamera(m_cameraDev); + } else { + emit error(QCamera::CameraError, QStringLiteral("Failed to find a camera")); + } +} + +bool QWasmCamera::setCameraFormat(const QCameraFormat &format) +{ + m_cameraFormat = format; + + return true; +} + +void QWasmCamera::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + QWasmMediaCaptureSession *captureSession = static_cast<QWasmMediaCaptureSession *>(session); + if (m_CaptureSession == captureSession) + return; + + m_CaptureSession = captureSession; +} + +void QWasmCamera::setFocusMode(QCamera::FocusMode mode) +{ + if (!isFocusModeSupported(mode)) + return; + + static constexpr std::string_view focusModeString = "focusMode"; + if (mode == QCamera::FocusModeManual) + m_cameraOutput->setDeviceSetting(focusModeString.data(), emscripten::val("manual")); + if (mode == QCamera::FocusModeAuto) + m_cameraOutput->setDeviceSetting(focusModeString.data(), emscripten::val("continuous")); + focusModeChanged(mode); +} + +bool QWasmCamera::isFocusModeSupported(QCamera::FocusMode mode) const +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val focusMode = caps["focusMode"]; + if (focusMode.isUndefined()) + return false; + + std::vector<std::string> focalModes; + + for (int i = 0; i < focusMode["length"].as<int>(); i++) + focalModes.push_back(focusMode[i].as<std::string>()); + + // Do we need to take into account focusDistance + // it is not always available, and what distance + // would be far/near + + bool found = false; + switch (mode) { + case QCamera::FocusModeAuto: + return std::find(focalModes.begin(), focalModes.end(), "continuous") != focalModes.end() + || std::find(focalModes.begin(), focalModes.end(), "single-shot") + != focalModes.end(); + case QCamera::FocusModeAutoNear: + case QCamera::FocusModeAutoFar: + case QCamera::FocusModeHyperfocal: + case QCamera::FocusModeInfinity: + break; + case QCamera::FocusModeManual: + found = std::find(focalModes.begin(), focalModes.end(), "manual") != focalModes.end(); + }; + return found; +} + +void QWasmCamera::setTorchMode(QCamera::TorchMode mode) +{ + if (!isTorchModeSupported(mode)) + return; + + if (m_wasmTorchMode == mode) + return; + + static constexpr std::string_view torchModeString = "torchMode"; + bool hasChanged = false; + switch (mode) { + case QCamera::TorchOff: + m_cameraOutput->setDeviceSetting(torchModeString.data(), emscripten::val(false)); + hasChanged = true; + break; + case QCamera::TorchOn: + m_cameraOutput->setDeviceSetting(torchModeString.data(), emscripten::val(true)); + hasChanged = true; + break; + case QCamera::TorchAuto: + break; + }; + m_wasmTorchMode = mode; + if (hasChanged) + torchModeChanged(m_wasmTorchMode); +} + +bool QWasmCamera::isTorchModeSupported(QCamera::TorchMode mode) const +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val exposureMode = caps["torch"]; + if (exposureMode.isUndefined()) + return false; + + return (mode != QCamera::TorchAuto); +} + +void QWasmCamera::setExposureMode(QCamera::ExposureMode mode) +{ + // TODO manually come up with exposureTime values ? + if (!isExposureModeSupported(mode)) + return; + + if (m_wasmExposureMode == mode) + return; + + bool hasChanged = false; + static constexpr std::string_view exposureModeString = "exposureMode"; + switch (mode) { + case QCamera::ExposureManual: + m_cameraOutput->setDeviceSetting(exposureModeString.data(), emscripten::val("manual")); + hasChanged = true; + break; + case QCamera::ExposureAuto: + m_cameraOutput->setDeviceSetting(exposureModeString.data(), emscripten::val("continuous")); + hasChanged = true; + break; + default: + break; + }; + + if (hasChanged) { + m_wasmExposureMode = mode; + exposureModeChanged(m_wasmExposureMode); + } +} + +bool QWasmCamera::isExposureModeSupported(QCamera::ExposureMode mode) const +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val exposureMode = caps["exposureMode"]; + if (exposureMode.isUndefined()) + return false; + + std::vector<std::string> exposureModes; + + for (int i = 0; i < exposureMode["length"].as<int>(); i++) + exposureModes.push_back(exposureMode[i].as<std::string>()); + + bool found = false; + switch (mode) { + case QCamera::ExposureAuto: + found = std::find(exposureModes.begin(), exposureModes.end(), "continuous") + != exposureModes.end(); + break; + case QCamera::ExposureManual: + found = std::find(exposureModes.begin(), exposureModes.end(), "manual") + != exposureModes.end(); + break; + default: + break; + }; + + return found; +} + +void QWasmCamera::setExposureCompensation(float bias) +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + emscripten::val exposureComp = caps["exposureCompensation"]; + if (exposureComp.isUndefined()) + return; + if (m_wasmExposureCompensation == bias) + return; + + static constexpr std::string_view exposureCompensationModeString = "exposureCompensation"; + m_cameraOutput->setDeviceSetting(exposureCompensationModeString.data(), emscripten::val(bias)); + m_wasmExposureCompensation = bias; + emit exposureCompensationChanged(m_wasmExposureCompensation); +} + +void QWasmCamera::setManualExposureTime(float secs) +{ + if (m_wasmExposureTime == secs) + return; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + emscripten::val exposureTime = caps["exposureTime"]; + if (exposureTime.isUndefined()) + return; + static constexpr std::string_view exposureTimeString = "exposureTime"; + m_cameraOutput->setDeviceSetting(exposureTimeString.data(), emscripten::val(secs)); + m_wasmExposureTime = secs; + emit exposureTimeChanged(m_wasmExposureTime); +} + +int QWasmCamera::isoSensitivity() const +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val isoSpeed = caps["iso"]; + if (isoSpeed.isUndefined()) + return 0; + + return isoSpeed.as<double>(); +} + +void QWasmCamera::setManualIsoSensitivity(int sens) +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + emscripten::val isoSpeed = caps["iso"]; + if (isoSpeed.isUndefined()) + return; + if (m_wasmIsoSensitivity == sens) + return; + static constexpr std::string_view isoString = "iso"; + m_cameraOutput->setDeviceSetting(isoString.data(), emscripten::val(sens)); + m_wasmIsoSensitivity = sens; + emit isoSensitivityChanged(m_wasmIsoSensitivity); +} + +bool QWasmCamera::isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val whiteBalanceMode = caps["whiteBalanceMode"]; + if (whiteBalanceMode.isUndefined()) + return false; + + if (mode == QCamera::WhiteBalanceAuto || mode == QCamera::WhiteBalanceManual) + return true; + + return false; +} + +void QWasmCamera::setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) +{ + if (!isWhiteBalanceModeSupported(mode)) + return; + + if (m_wasmWhiteBalanceMode == mode) + return; + + bool hasChanged = false; + static constexpr std::string_view whiteBalanceModeString = "whiteBalanceMode"; + switch (mode) { + case QCamera::WhiteBalanceAuto: + m_cameraOutput->setDeviceSetting(whiteBalanceModeString.data(), emscripten::val("auto")); + hasChanged = true; + break; + case QCamera::WhiteBalanceManual: + m_cameraOutput->setDeviceSetting(whiteBalanceModeString.data(), emscripten::val("manual")); + hasChanged = true; + break; + default: + break; + }; + + if (hasChanged) { + m_wasmWhiteBalanceMode = mode; + emit whiteBalanceModeChanged(m_wasmWhiteBalanceMode); + } +} + +void QWasmCamera::setColorTemperature(int temperature) +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + emscripten::val whiteBalanceMode = caps["colorTemperature"]; + if (whiteBalanceMode.isUndefined()) + return; + if(m_wasmColorTemperature == temperature) + return; + + static constexpr std::string_view colorBalanceString = "colorTemperature"; + m_cameraOutput->setDeviceSetting(colorBalanceString.data(), emscripten::val(temperature)); + m_wasmColorTemperature = temperature; + colorTemperatureChanged(m_wasmColorTemperature); +} + +void QWasmCamera::createCamera(const QCameraDevice &camera) +{ + m_cameraOutput->addCameraSourceElement(camera.id().toStdString()); +} + +void QWasmCamera::updateCameraFeatures() +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + QCamera::Features cameraFeatures; + + if (!caps["colorTemperature"].isUndefined()) + cameraFeatures |= QCamera::Feature::ColorTemperature; + + if (!caps["exposureCompensation"].isUndefined()) + cameraFeatures |= QCamera::Feature::ExposureCompensation; + + if (!caps["iso"].isUndefined()) + cameraFeatures |= QCamera::Feature::IsoSensitivity; + + if (!caps["exposureTime"].isUndefined()) + cameraFeatures |= QCamera::Feature::ManualExposureTime; + + if (!caps["focusDistance"].isUndefined()) + cameraFeatures |= QCamera::Feature::FocusDistance; + + supportedFeaturesChanged(cameraFeatures); +} diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmcamera_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera_p.h new file mode 100644 index 000000000..6f8040dbf --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera_p.h @@ -0,0 +1,95 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMCAMERA_H +#define QWASMCAMERA_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformcamera_p.h> +#include <private/qplatformvideodevices_p.h> +#include <common/qwasmvideooutput_p.h> + +#include <QCameraDevice> +#include <QtCore/qloggingcategory.h> + +#include <emscripten/val.h> +#include <emscripten/bind.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmCamera) + +class QWasmMediaCaptureSession; + +class QWasmCamera : public QPlatformCamera +{ + Q_OBJECT + +public: + explicit QWasmCamera(QCamera *camera); + ~QWasmCamera(); + + bool isActive() const override; + void setActive(bool active) override; + + void setCamera(const QCameraDevice &camera) override; + bool setCameraFormat(const QCameraFormat &format) override; + + void setCaptureSession(QPlatformMediaCaptureSession *session) override; + + void setFocusMode(QCamera::FocusMode mode) override; + bool isFocusModeSupported(QCamera::FocusMode mode) const override; + + void setTorchMode(QCamera::TorchMode mode) override; + bool isTorchModeSupported(QCamera::TorchMode mode) const override; + + void setExposureMode(QCamera::ExposureMode mode) override; + bool isExposureModeSupported(QCamera::ExposureMode mode) const override; + void setExposureCompensation(float bias) override; + + void setManualExposureTime(float) override; + int isoSensitivity() const override; + void setManualIsoSensitivity(int) override; + + bool isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const override; + void setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) override; + + void setColorTemperature(int temperature) override; + + QWasmVideoOutput *cameraOutput() { return m_cameraOutput.data(); } + +private: + void createCamera(const QCameraDevice &camera); + void updateCameraFeatures(); + + QCameraDevice m_cameraDev; + QWasmMediaCaptureSession *m_CaptureSession; + bool m_cameraActive = false; + QScopedPointer <QWasmVideoOutput> m_cameraOutput; + + emscripten::val supportedCapabilities = emscripten::val::object(); // browser + emscripten::val currentCapabilities = emscripten::val::object(); // camera track + emscripten::val currentSettings = emscripten::val::object(); // camera track + + QCamera::TorchMode m_wasmTorchMode; + QCamera::ExposureMode m_wasmExposureMode; + float m_wasmExposureTime; + float m_wasmExposureCompensation; + int m_wasmIsoSensitivity; + QCamera::WhiteBalanceMode m_wasmWhiteBalanceMode; + int m_wasmColorTemperature; +}; + +QT_END_NAMESPACE + +#endif // QWASMCAMERA_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture.cpp new file mode 100644 index 000000000..f62d6f1a6 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture.cpp @@ -0,0 +1,130 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmimagecapture_p.h" +#include <qimagewriter.h> +#include "qwasmmediacapturesession_p.h" +#include "qwasmcamera_p.h" +#include "qwasmvideosink_p.h" + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(qWasmImageCapture, "qt.multimedia.wasm.imagecapture") +/* TODO +signals: +imageExposed +*/ +QWasmImageCapture::QWasmImageCapture(QImageCapture *parent) : QPlatformImageCapture(parent) { } + +QWasmImageCapture::~QWasmImageCapture() = default; + +int QWasmImageCapture::capture(const QString &fileName) +{ + if (!isReadyForCapture()) { + emit error(m_lastId, QImageCapture::NotReadyError, msgCameraNotReady()); + return -1; + } + + // TODO if fileName.isEmpty() we choose filename and location + + QImage image = takePicture(); + if (image.isNull()) + return -1; + + QImageWriter writer(fileName); + // TODO + // writer.setQuality(quality); + // writer.setFormat("png"); + + if (writer.write(image)) { + qCDebug(qWasmImageCapture) << Q_FUNC_INFO << "image saved"; + emit imageSaved(m_lastId, fileName); + } else { + QImageCapture::Error err = (writer.error() == QImageWriter::UnsupportedFormatError) + ? QImageCapture::FormatError + : QImageCapture::ResourceError; + + emit error(m_lastId, err, writer.errorString()); + } + + return m_lastId; +} + +int QWasmImageCapture::captureToBuffer() +{ + if (!isReadyForCapture()) { + emit error(m_lastId, QImageCapture::NotReadyError, msgCameraNotReady()); + return -1; + } + + QImage image = takePicture(); + if (image.isNull()) + return -1; + + emit imageCaptured(m_lastId, image); + return m_lastId; +} + +QImage QWasmImageCapture::takePicture() +{ + QVideoFrame thisFrame = m_captureSession->videoSink()->videoFrame(); + if (!thisFrame.isValid()) + return QImage(); + + m_lastId++; + emit imageAvailable(m_lastId, thisFrame); + + QImage image = thisFrame.toImage(); + if (image.isNull()) { + qCDebug(qWasmImageCapture) << Q_FUNC_INFO << "image is null"; + emit error(m_lastId, QImageCapture::ResourceError, QStringLiteral("Resource error")); + return QImage(); + } + + emit imageCaptured(m_lastId, image); + if (m_settings.resolution().isValid() && m_settings.resolution() != image.size()) + image = image.scaled(m_settings.resolution()); + + return image; +} + +bool QWasmImageCapture::isReadyForCapture() const +{ + return m_isReadyForCapture; +} + +QImageEncoderSettings QWasmImageCapture::imageSettings() const +{ + return m_settings; +} + +void QWasmImageCapture::setImageSettings(const QImageEncoderSettings &settings) +{ + m_settings = settings; +} + +void QWasmImageCapture::setReadyForCapture(bool isReady) +{ + if (m_isReadyForCapture != isReady) { + m_isReadyForCapture = isReady; + emit readyForCaptureChanged(m_isReadyForCapture); + } +} + +void QWasmImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + QWasmMediaCaptureSession *captureSession = static_cast<QWasmMediaCaptureSession *>(session); + // nullptr clears + if (m_captureSession == captureSession) + return; + + m_isReadyForCapture = captureSession; + if (captureSession) { + m_lastId = 0; + m_captureSession = captureSession; + } + emit readyForCaptureChanged(m_isReadyForCapture); + m_captureSession = captureSession; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture_p.h new file mode 100644 index 000000000..2e9e9b227 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture_p.h @@ -0,0 +1,58 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMIMAGECAPTURE_H +#define QWASMIMAGECAPTURE_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QObject> +#include <private/qplatformimagecapture_p.h> +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmImageCapture) + +class QWasmMediaCaptureSession; + +class QWasmImageCapture : public QPlatformImageCapture +{ + Q_OBJECT +public: + explicit QWasmImageCapture(QImageCapture *parent = nullptr); + ~QWasmImageCapture(); + + bool isReadyForCapture() const override; + + int capture(const QString &fileName) override; + int captureToBuffer() override; + + QImageEncoderSettings imageSettings() const override; + void setImageSettings(const QImageEncoderSettings &settings) override; + + void setReadyForCapture(bool isReady); + + void setCaptureSession(QPlatformMediaCaptureSession *session); + +private: + QImage takePicture(); + + // weak + QWasmMediaCaptureSession *m_captureSession = nullptr; + QImageEncoderSettings m_settings; + bool m_isReadyForCapture = false; + int m_lastId = 0; +}; + +QT_END_NAMESPACE +#endif // QWASMIMAGECAPTURE_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession.cpp new file mode 100644 index 000000000..52d166a9e --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession.cpp @@ -0,0 +1,95 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediacapturesession_p.h" +#include "mediacapture/qwasmimagecapture_p.h" + +#include "qwasmcamera_p.h" + +Q_LOGGING_CATEGORY(qWasmMediaCaptureSession, "qt.multimedia.wasm.capturesession") + +QWasmMediaCaptureSession::QWasmMediaCaptureSession() = default; + +QWasmMediaCaptureSession::~QWasmMediaCaptureSession() = default; + +QPlatformCamera *QWasmMediaCaptureSession::camera() +{ + return m_camera.data(); +} + +void QWasmMediaCaptureSession::setCamera(QPlatformCamera *camera) +{ + if (!camera) + return; + QWasmCamera *wasmCamera = static_cast<QWasmCamera *>(camera); + if (!wasmCamera || m_camera.data() == wasmCamera) + return; + m_camera.reset(wasmCamera); + emit cameraChanged(); + m_camera->setCaptureSession(this); +} + +QPlatformImageCapture *QWasmMediaCaptureSession::imageCapture() +{ + return m_imageCapture; +} + +void QWasmMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCapture) +{ + if (m_imageCapture == imageCapture) + return; + + if (m_imageCapture) + m_imageCapture->setCaptureSession(nullptr); + + m_imageCapture = static_cast<QWasmImageCapture *>(imageCapture); + + if (m_imageCapture) { + m_imageCapture->setCaptureSession(this); + + m_imageCapture->setReadyForCapture(true); + emit imageCaptureChanged(); + } +} + +QPlatformMediaRecorder *QWasmMediaCaptureSession::mediaRecorder() +{ + return m_mediaRecorder; +} + +void QWasmMediaCaptureSession::setMediaRecorder(QPlatformMediaRecorder *mediaRecorder) +{ + if (m_mediaRecorder == mediaRecorder) + return; + + if (m_mediaRecorder) + m_mediaRecorder->setCaptureSession(nullptr); + + m_mediaRecorder = static_cast<QWasmMediaRecorder *>(mediaRecorder); + + if (m_mediaRecorder) + m_mediaRecorder->setCaptureSession(this); +} + +void QWasmMediaCaptureSession::setAudioInput(QPlatformAudioInput *input) +{ + if (m_audioInput == input) + return; + + m_needsAudio = !input; + m_audioInput = input; +} + +void QWasmMediaCaptureSession::setVideoPreview(QVideoSink *sink) +{ + if (m_wasmSink == sink) + return; + m_wasmSink = sink; +} + +void QWasmMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output) +{ + if (m_audioOutput == output) + return; + m_audioOutput = output; +} diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession_p.h new file mode 100644 index 000000000..e21147917 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession_p.h @@ -0,0 +1,70 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMMEDIACAPTURESESSION_H +#define QWASMMEDIACAPTURESESSION_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qwasmimagecapture_p.h" + +#include <private/qplatformmediacapture_p.h> +#include <private/qplatformmediaintegration_p.h> +#include "qwasmmediarecorder_p.h" +#include <QScopedPointer> +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmMediaCaptureSession) + +class QAudioInput; +class QWasmCamera; + +class QWasmMediaCaptureSession : public QPlatformMediaCaptureSession +{ +public: + explicit QWasmMediaCaptureSession(); + ~QWasmMediaCaptureSession() override; + + QPlatformCamera *camera() override; + void setCamera(QPlatformCamera *camera) override; + + QPlatformImageCapture *imageCapture() override; + void setImageCapture(QPlatformImageCapture *imageCapture) override; + + QPlatformMediaRecorder *mediaRecorder() override; + void setMediaRecorder(QPlatformMediaRecorder *recorder) override; + + void setAudioInput(QPlatformAudioInput *input) override; + void setVideoPreview(QVideoSink *sink) override; + void setAudioOutput(QPlatformAudioOutput *output) override; + + bool hasAudio() { return m_needsAudio; } + QVideoSink *videoSink() { return m_wasmSink; } + +private: + QWasmMediaRecorder *m_mediaRecorder = nullptr; + + QScopedPointer <QWasmCamera> m_camera; + + QWasmImageCapture *m_imageCapture = nullptr; + + QPlatformAudioInput *m_audioInput = nullptr; + QPlatformAudioOutput *m_audioOutput = nullptr; + bool m_needsAudio = false; + QVideoSink *m_wasmSink = nullptr; +}; + +QT_END_NAMESPACE + +#endif // QWASMMEDIACAPTURESESSION_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp new file mode 100644 index 000000000..7adc5ee96 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp @@ -0,0 +1,417 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediarecorder_p.h" +#include "qwasmmediacapturesession_p.h" +#include "qwasmcamera_p.h" +#include <private/qstdweb_p.h> +#include <QtCore/QIODevice> +#include <QFile> +#include <QTimer> + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(qWasmMediaRecorder, "qt.multimedia.wasm.mediarecorder") + +QWasmMediaRecorder::QWasmMediaRecorder(QMediaRecorder *parent) + : QPlatformMediaRecorder(parent) +{ + m_durationTimer.reset(new QElapsedTimer()); +} + +QWasmMediaRecorder::~QWasmMediaRecorder() +{ + if (m_outputTarget->isOpen()) + m_outputTarget->close(); + + if (!m_mediaRecorder.isNull()) { + m_mediaStreamDataAvailable.reset(nullptr); + m_mediaStreamStopped.reset(nullptr); + m_mediaStreamError.reset(nullptr); + m_mediaStreamStart.reset(nullptr); + } +} + +bool QWasmMediaRecorder::isLocationWritable(const QUrl &location) const +{ + return location.isValid() && (location.isLocalFile() || location.isRelative()); +} + +QMediaRecorder::RecorderState QWasmMediaRecorder::state() const +{ + QMediaRecorder::RecorderState recordingState = QMediaRecorder::StoppedState; + + if (!m_mediaRecorder.isUndefined()) { + std::string state = m_mediaRecorder["state"].as<std::string>(); + if (state == "recording") + recordingState = QMediaRecorder::RecordingState; + else if (state == "paused") + recordingState = QMediaRecorder::PausedState; + } + return recordingState; +} + +qint64 QWasmMediaRecorder::duration() const +{ // milliseconds + return m_durationMs; +} + +void QWasmMediaRecorder::record(QMediaEncoderSettings &settings) +{ + if (!m_session) + return; + + m_mediaSettings = settings; + initUserMedia(); +} + +void QWasmMediaRecorder::pause() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + m_mediaRecorder.call<void>("pause"); + emit stateChanged(state()); +} + +void QWasmMediaRecorder::resume() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + + m_mediaRecorder.call<void>("resume"); + emit stateChanged(state()); +} + +void QWasmMediaRecorder::stop() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + if (m_mediaRecorder["state"].as<std::string>() == "recording") + m_mediaRecorder.call<void>("stop"); +} + +void QWasmMediaRecorder::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + m_session = static_cast<QWasmMediaCaptureSession *>(session); +} + +bool QWasmMediaRecorder::hasCamera() const +{ + return m_session && m_session->camera(); +} + +void QWasmMediaRecorder::initUserMedia() +{ + setUpFileSink(); + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + QWasmCamera *wasmCamera = reinterpret_cast<QWasmCamera *>(m_session->camera()); + emscripten::val m_video = wasmCamera->cameraOutput()->surfaceElement(); + if (m_video.isNull() || m_video.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "video element not found"; + return; + } + + emscripten::val stream = m_video["srcObject"]; + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "video stream not found"; + return; + } + setStream(stream); +} + +void QWasmMediaRecorder::startAudioRecording() +{ + startStream(); +} + +void QWasmMediaRecorder::setStream(emscripten::val stream) +{ + emscripten::val emMediaSettings = emscripten::val::object(); + QMediaFormat::VideoCodec videoCodec = m_mediaSettings.videoCodec(); + QMediaFormat::AudioCodec audioCodec = m_mediaSettings.audioCodec(); + QMediaFormat::FileFormat fileFormat = m_mediaSettings.fileFormat(); + + // mime and codecs + QString mimeCodec; + if (!m_mediaSettings.mimeType().name().isEmpty()) { + mimeCodec = m_mediaSettings.mimeType().name(); + + if (videoCodec != QMediaFormat::VideoCodec::Unspecified) + mimeCodec += QStringLiteral(": codecs="); + + if (audioCodec != QMediaFormat::AudioCodec::Unspecified) { + // TODO + } + + if (fileFormat != QMediaFormat::UnspecifiedFormat) + mimeCodec += QMediaFormat::fileFormatName(m_mediaSettings.fileFormat()); + + emMediaSettings.set("mimeType", mimeCodec.toStdString()); + } + + if (m_mediaSettings.audioBitRate() > 0) + emMediaSettings.set("audioBitsPerSecond", emscripten::val(m_mediaSettings.audioBitRate())); + + if (m_mediaSettings.videoBitRate() > 0) + emMediaSettings.set("videoBitsPerSecond", emscripten::val(m_mediaSettings.videoBitRate())); + + // create the MediaRecorder, and set up data callback + m_mediaRecorder = emscripten::val::global("MediaRecorder").new_(stream, emMediaSettings); + + m_mediaRecorder.set("data-mediarecordercontext", + emscripten::val(quintptr(reinterpret_cast<void *>(this)))); + + // dataavailable + auto callback = [](emscripten::val blob) { + if (blob.isUndefined() || blob.isNull()) { + qCDebug(qWasmMediaRecorder) << "blob is null"; + return; + } + + if (blob["target"].isUndefined() || blob["target"].isNull()) + return; + if (blob["data"].isUndefined() || blob["data"].isNull()) + return; + if (blob["target"]["data-mediarecordercontext"].isUndefined() + || blob["target"]["data-mediarecordercontext"].isNull()) + return; + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + blob["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + const double timeCode = blob.hasOwnProperty("timecode") + ? blob["timecode"].as<double>() : 0; + recorder->audioDataAvailable(blob["data"],timeCode); + } + }; + + m_mediaStreamDataAvailable.reset( + new qstdweb::EventCallback(m_mediaRecorder, "dataavailable", callback)); + + // stopped + auto stoppedCallback = [](emscripten::val event) { + + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = false; + recorder->m_durationTimer->invalidate(); + + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamStopped.reset( + new qstdweb::EventCallback(m_mediaRecorder, "stop", stoppedCallback)); + + // error + auto errorCallback = [](emscripten::val theError) { + + if (theError.isUndefined() || theError.isNull()) { + qCDebug(qWasmMediaRecorder) << "error is null"; + return; + } + qCDebug(qWasmMediaRecorder) + << theError["code"].as<int>() + << QString::fromStdString(theError["message"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + theError["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->error(QMediaRecorder::ResourceError, QString::fromStdString(theError["message"].as<std::string>())); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamError.reset( + new qstdweb::EventCallback(m_mediaRecorder, "error", errorCallback)); + + // start + auto startCallback = [](emscripten::val eventVal) { + + if (eventVal.isUndefined() || eventVal.isNull()) { + qCDebug(qWasmMediaRecorder) << "eventVal is null"; + return; + } + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + eventVal["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamStart.reset( + new qstdweb::EventCallback(m_mediaRecorder, "start", startCallback)); + + // set up what options we can + setTrackContraints(m_mediaSettings, stream); +} + +void QWasmMediaRecorder::audioDataAvailable(emscripten::val blob, double timeCodeDifference) +{ + Q_UNUSED(timeCodeDifference) + if (blob.isUndefined() || blob.isNull()) { + qCDebug(qWasmMediaRecorder) << "blob is null"; + return; + } + + auto fileReader = std::make_shared<qstdweb::FileReader>(); + + fileReader->onError( + [=](emscripten::val theError) { + + error(QMediaRecorder::ResourceError, QString::fromStdString(theError["message"].as<std::string>())); + }); + + fileReader->onAbort( + [=](emscripten::val ) { + error(QMediaRecorder::ResourceError, QStringLiteral("File read aborted")); + }); + + fileReader->onLoad( + [=](emscripten::val) { + + if (fileReader->val().isNull() || fileReader->val().isUndefined()) + return; + qstdweb::ArrayBuffer result = fileReader->result(); + if (result.val().isNull() || result.val().isUndefined()) + return; + QByteArray fileContent = qstdweb::Uint8Array(result).copyToQByteArray(); + + if (m_isRecording && !fileContent.isEmpty()) { + m_durationMs = m_durationTimer->elapsed(); + if (m_outputTarget->isOpen()) + m_outputTarget->write(fileContent, fileContent.length()); + // we've read everything + emit durationChanged(m_durationMs); + } + }); + + fileReader->readAsArrayBuffer(qstdweb::Blob(blob)); +} + +// constraints are suggestions, as not all hardware supports all settings +void QWasmMediaRecorder::setTrackContraints(QMediaEncoderSettings &settings, emscripten::val stream) +{ + if (stream.isUndefined() || stream.isNull()) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaStream"; + return; + } + + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + // check which ones are supported + // emscripten::val allConstraints = mediaDevices.call<emscripten::val>("getSupportedConstraints"); + // browsers only support some settings + + emscripten::val videoParams = emscripten::val::object(); + emscripten::val constraints = emscripten::val::object(); + + if (hasCamera()) { + if (settings.videoFrameRate() > 0) + videoParams.set("frameRate", emscripten::val(settings.videoFrameRate())); + if (settings.videoResolution().height() > 0) + videoParams.set("height", + emscripten::val(settings.videoResolution().height())); // viewportHeight? + if (settings.videoResolution().width() > 0) + videoParams.set("width", emscripten::val(settings.videoResolution().width())); + + constraints.set("video", videoParams); // only video here + } + + emscripten::val audioParams = emscripten::val::object(); + if (settings.audioSampleRate() > 0) + audioParams.set("sampleRate", emscripten::val(settings.audioSampleRate())); // may not work + if (settings.audioBitRate() > 0) + audioParams.set("sampleSize", emscripten::val(settings.audioBitRate())); // may not work + if (settings.audioChannelCount() > 0) + audioParams.set("channelCount", emscripten::val(settings.audioChannelCount())); + + constraints.set("audio", audioParams); // only audio here + + if (hasCamera() && stream["active"].as<bool>()) { + emscripten::val videoTracks = emscripten::val::undefined(); + videoTracks = stream.call<emscripten::val>("getVideoTracks"); + if (videoTracks.isNull() || videoTracks.isUndefined()) { + qCDebug(qWasmMediaRecorder) << "no video tracks"; + return; + } + if (videoTracks["length"].as<int>() > 0) { + // try to apply the video options + qstdweb::Promise::make(videoTracks[0], QStringLiteral("applyConstraints"), + { .thenFunc = + [=](emscripten::val result) { + Q_UNUSED(result) + startStream(); + }, + .catchFunc = + [=](emscripten::val theError) { + qWarning() << "setting video params failed error"; + qCDebug(qWasmMediaRecorder) + << theError["code"].as<int>() + << QString::fromStdString(theError["message"].as<std::string>()); + error(QMediaRecorder::ResourceError, QString::fromStdString(theError["message"].as<std::string>())); + } }, + constraints); + } + } +} + +// this starts the recording stream +void QWasmMediaRecorder::startStream() +{ + if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "could not find MediaStream"; + return; + } + + constexpr int sliceSizeInMs = 250; // TODO find what size is best + m_mediaRecorder.call<void>("start", emscripten::val(sliceSizeInMs)); + + /* this method can optionally be passed a timeslice argument with a value in milliseconds. + * If this is specified, the media will be captured in separate chunks of that duration, + * rather than the default behavior of recording the media in a single large chunk.*/ + + emit stateChanged(state()); +} + +void QWasmMediaRecorder::setUpFileSink() +{ + QString m_targetFileName = outputLocation().toLocalFile(); + if (m_targetFileName.isEmpty()) { + if (hasCamera()) { + m_targetFileName = "/home/web_user/tmp.mp4"; + } else { + m_targetFileName = "/home/web_user/tmp.mp3"; + } + QPlatformMediaRecorder::setOutputLocation(m_targetFileName); + } + + m_outputTarget = new QFile(m_targetFileName, this); + if (!m_outputTarget->open(QIODevice::WriteOnly)) { + qWarning() << "target file is not writable"; + return; + } +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder_p.h new file mode 100644 index 000000000..7d707729d --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder_p.h @@ -0,0 +1,84 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMMEDIARECORDER_H +#define QWASMMEDIARECORDER_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformmediarecorder_p.h> +#include <private/qplatformmediacapture_p.h> +#include <QtCore/qglobal.h> +#include <QtCore/qloggingcategory.h> +#include <QElapsedTimer> + +#include <emscripten.h> +#include <emscripten/val.h> +#include <emscripten/bind.h> +#include <private/qstdweb_p.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmMediaRecorder) + +class QWasmMediaCaptureSession; +class QIODevice; + +class QWasmMediaRecorder final : public QObject, public QPlatformMediaRecorder +{ + Q_OBJECT +public: + explicit QWasmMediaRecorder(QMediaRecorder *parent); + ~QWasmMediaRecorder() final; + + bool isLocationWritable(const QUrl &location) const override; + QMediaRecorder::RecorderState state() const override; + qint64 duration() const override; + void record(QMediaEncoderSettings &settings) override; + void pause() override; + void resume() override; + void stop() override; + + void setCaptureSession(QPlatformMediaCaptureSession *session); + +private: + + bool hasCamera() const; + void startAudioRecording(); + void setStream(emscripten::val stream); + void streamCallback(emscripten::val event); + void exceptionCallback(emscripten::val event); + void dataAvailableCallback(emscripten::val dataEvent); + void startStream(); + void setTrackContraints(QMediaEncoderSettings &settings, emscripten::val stream); + void initUserMedia(); + void audioDataAvailable(emscripten::val Blob, double timeCodeDifference); + void setUpFileSink(); + + emscripten::val m_mediaRecorder = emscripten::val::undefined(); + QWasmMediaCaptureSession *m_session = nullptr; + QMediaEncoderSettings m_mediaSettings; + QIODevice *m_outputTarget; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamDataAvailable; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamStopped; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamError; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamStart; + qint64 m_durationMs = 0; + bool m_isRecording = false; + QScopedPointer <QElapsedTimer> m_durationTimer; + +private Q_SLOTS: +}; + +QT_END_NAMESPACE + +#endif // QWASMMEDIARECORDER_H diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp index 87e575360..1e30b092e 100644 --- a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp +++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp @@ -19,6 +19,7 @@ QWasmMediaPlayer::QWasmMediaPlayer(QMediaPlayer *parent) m_videoOutput(new QWasmVideoOutput), m_State(QWasmMediaPlayer::Idle) { + qCDebug(lcMediaPlayer) << Q_FUNC_INFO << this; initVideo(); } @@ -31,10 +32,12 @@ void QWasmMediaPlayer::initVideo() { m_videoOutput->setVideoMode(QWasmVideoOutput::VideoOutput); QUuid videoElementId = QUuid::createUuid(); + qCDebug(lcMediaPlayer) << Q_FUNC_INFO << "videoElementId"<< videoElementId << this; + m_videoOutput->createVideoElement(videoElementId.toString(QUuid::WithoutBraces).toStdString()); m_videoOutput->doElementCallbacks(); m_videoOutput->createOffscreenElement(QSize(1280, 720)); - m_videoOutput->updateVideoElementGeometry(QRect(0, 0, 320, 240)); // initial size + m_videoOutput->updateVideoElementGeometry(QRect(0, 0, 1280, 720)); // initial size 720p standard connect(m_videoOutput, &QWasmVideoOutput::bufferingChanged, this, &QWasmMediaPlayer::bufferingChanged); diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h index 754f502ab..f48b0b42b 100644 --- a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h +++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h @@ -61,21 +61,17 @@ public: QUrl media() const override; const QIODevice *mediaStream() const override; void setMedia(const QUrl &mediaContent, QIODevice *stream) override; - void setVideoSink(QVideoSink *surface) override; - void setAudioOutput(QPlatformAudioOutput *output) override; - void updateAudioDevice(); - void setPosition(qint64 position) override; void play() override; void pause() override; void stop() override; - bool isSeekable() const override; - int trackCount(TrackType trackType) override; + void updateAudioDevice(); + private Q_SLOTS: void volumeChanged(float volume); void mutedChanged(bool muted); diff --git a/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp b/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp index b4d0a7e5f..2458bdc7c 100644 --- a/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp +++ b/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp @@ -4,6 +4,9 @@ #include "qwasmmediaintegration_p.h" #include <QLoggingCategory> +#include <QCamera> +#include <QCameraDevice> + #include <private/qplatformmediaformatinfo_p.h> #include <private/qplatformmediaplugin_p.h> #include <private/qplatformmediadevices_p.h> @@ -14,6 +17,12 @@ #include "qwasmaudioinput_p.h" #include "common/qwasmaudiooutput_p.h" +#include "mediacapture/qwasmmediacapturesession_p.h" +#include "mediacapture/qwasmmediarecorder_p.h" +#include "mediacapture/qwasmcamera_p.h" +#include "mediacapture/qwasmmediacapturesession_p.h" +#include "mediacapture/qwasmimagecapture_p.h" + QT_BEGIN_NAMESPACE @@ -39,6 +48,7 @@ public: QWasmMediaIntegration::QWasmMediaIntegration() { + m_videoDevices = std::make_unique<QWasmCameraDevices>(this); } QWasmMediaIntegration::~QWasmMediaIntegration() @@ -74,6 +84,32 @@ QPlatformMediaFormatInfo *QWasmMediaIntegration::formatInfo() return m_formatInfo; } +QMaybe<QPlatformMediaCaptureSession *> QWasmMediaIntegration::createCaptureSession() +{ + return new QWasmMediaCaptureSession(); +} + +QMaybe<QPlatformMediaRecorder *> QWasmMediaIntegration::createRecorder(QMediaRecorder *recorder) +{ + return new QWasmMediaRecorder(recorder); +} + +QMaybe<QPlatformCamera *> QWasmMediaIntegration::createCamera(QCamera *camera) +{ + return new QWasmCamera(camera); +} + +QMaybe<QPlatformImageCapture *> +QWasmMediaIntegration::createImageCapture(QImageCapture *imageCapture) +{ + return new QWasmImageCapture(imageCapture); +} + +QList<QCameraDevice> QWasmMediaIntegration::videoInputs() +{ + return m_videoDevices->videoDevices(); +} + QT_END_NAMESPACE #include "qwasmmediaintegration.moc" diff --git a/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h b/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h index bc86923ba..75af8a731 100644 --- a/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h +++ b/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h @@ -17,6 +17,8 @@ #include <private/qplatformmediaintegration_p.h> +#include <private/qwasmmediadevices_p.h> + QT_BEGIN_NAMESPACE class QWasmMediaDevices; @@ -36,6 +38,12 @@ public: QMaybe<QPlatformAudioInput *> createAudioInput(QAudioInput *audioInput) override; QMaybe<QPlatformAudioOutput *> createAudioOutput(QAudioOutput *q) override; + + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformCamera *> createCamera(QCamera *camera) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *recorder) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *imageCapture) override; + QList<QCameraDevice> videoInputs() override; }; QT_END_NAMESPACE |