diff options
Diffstat (limited to 'src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp')
| -rw-r--r-- | src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp | 520 |
1 files changed, 520 insertions, 0 deletions
diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp new file mode 100644 index 000000000..98f04616a --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp @@ -0,0 +1,520 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediarecorder_p.h" +#include "qwasmmediacapturesession_p.h" +#include <private/qplatformmediadevices_p.h> +#include <private/qplatformmediaintegration_p.h> +#include "qwasmcamera_p.h" +#include "qwasmaudioinput_p.h" + +#include <private/qstdweb_p.h> +#include <QtCore/QIODevice> +#include <QFile> +#include <QTimer> +#include <QDebug> + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(qWasmMediaRecorder, "qt.multimedia.wasm.mediarecorder") + +QWasmMediaRecorder::QWasmMediaRecorder(QMediaRecorder *parent) + : QPlatformMediaRecorder(parent) +{ + m_durationTimer.reset(new QElapsedTimer()); + QPlatformMediaIntegration::instance()->mediaDevices(); // initialize getUserMedia +} + +QWasmMediaRecorder::~QWasmMediaRecorder() +{ + if (m_outputTarget->isOpen()) + m_outputTarget->close(); + + if (!m_mediaRecorder.isNull()) { + m_mediaStreamDataAvailable.reset(nullptr); + m_mediaStreamStopped.reset(nullptr); + m_mediaStreamError.reset(nullptr); + m_mediaStreamStart.reset(nullptr); + } +} + +bool QWasmMediaRecorder::isLocationWritable(const QUrl &location) const +{ + return location.isValid() && (location.isLocalFile() || location.isRelative()); +} + +QMediaRecorder::RecorderState QWasmMediaRecorder::state() const +{ + QMediaRecorder::RecorderState recordingState = QMediaRecorder::StoppedState; + + if (!m_mediaRecorder.isUndefined()) { + std::string state = m_mediaRecorder["state"].as<std::string>(); + if (state == "recording") + recordingState = QMediaRecorder::RecordingState; + else if (state == "paused") + recordingState = QMediaRecorder::PausedState; + } + return recordingState; +} + +qint64 QWasmMediaRecorder::duration() const +{ // milliseconds + return m_durationMs; +} + +void QWasmMediaRecorder::record(QMediaEncoderSettings &settings) +{ + if (!m_session) + return; + + m_mediaSettings = settings; + initUserMedia(); +} + +void QWasmMediaRecorder::pause() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + m_mediaRecorder.call<void>("pause"); + emit stateChanged(state()); +} + +void QWasmMediaRecorder::resume() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + + m_mediaRecorder.call<void>("resume"); + emit stateChanged(state()); +} + +void QWasmMediaRecorder::stop() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + if (m_mediaRecorder["state"].as<std::string>() == "recording") + m_mediaRecorder.call<void>("stop"); +} + +void QWasmMediaRecorder::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + m_session = static_cast<QWasmMediaCaptureSession *>(session); +} + +bool QWasmMediaRecorder::hasCamera() const +{ + return m_session && m_session->camera(); +} + +void QWasmMediaRecorder::initUserMedia() +{ + setUpFileSink(); + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + if (mediaDevices.isNull() || mediaDevices.isUndefined()) { + qCDebug(qWasmMediaRecorder) << "MediaDevices are undefined or null"; + return; + } + + if (!m_session) + return; + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << m_session; + + emscripten::val stream = emscripten::val::undefined(); + if (hasCamera()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "has camera"; + QWasmCamera *wasmCamera = reinterpret_cast<QWasmCamera *>(m_session->camera()); + + if (wasmCamera) { + emscripten::val m_video = wasmCamera->cameraOutput()->surfaceElement(); + if (m_video.isNull() || m_video.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "video element not found"; + return; + } + + stream = m_video["srcObject"]; + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "Video input stream not found"; + return; + } + } + } else { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "has audio"; + stream = static_cast<QWasmAudioInput *>(m_session->audioInput())->mediaStream(); + + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "Audio input stream not found"; + return; + } + } + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "No input stream found"; + return; + } + + setStream(stream); +} + +void QWasmMediaRecorder::startAudioRecording() +{ + startStream(); +} + +void QWasmMediaRecorder::setStream(emscripten::val stream) +{ + emscripten::val emMediaSettings = emscripten::val::object(); + QMediaFormat::VideoCodec videoCodec = m_mediaSettings.videoCodec(); + QMediaFormat::AudioCodec audioCodec = m_mediaSettings.audioCodec(); + QMediaFormat::FileFormat fileFormat = m_mediaSettings.fileFormat(); + + // mime and codecs + QString mimeCodec; + if (!m_mediaSettings.mimeType().name().isEmpty()) { + mimeCodec = m_mediaSettings.mimeType().name(); + + if (videoCodec != QMediaFormat::VideoCodec::Unspecified) + mimeCodec += QStringLiteral(": codecs="); + + if (audioCodec != QMediaFormat::AudioCodec::Unspecified) { + // TODO + } + + if (fileFormat != QMediaFormat::UnspecifiedFormat) + mimeCodec += QMediaFormat::fileFormatName(m_mediaSettings.fileFormat()); + + emMediaSettings.set("mimeType", mimeCodec.toStdString()); + } + + if (m_mediaSettings.audioBitRate() > 0) + emMediaSettings.set("audioBitsPerSecond", emscripten::val(m_mediaSettings.audioBitRate())); + + if (m_mediaSettings.videoBitRate() > 0) + emMediaSettings.set("videoBitsPerSecond", emscripten::val(m_mediaSettings.videoBitRate())); + + // create the MediaRecorder, and set up data callback + m_mediaRecorder = emscripten::val::global("MediaRecorder").new_(stream, emMediaSettings); + + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "m_mediaRecorder state:" + << QString::fromStdString(m_mediaRecorder["state"].as<std::string>()); + + if (m_mediaRecorder.isNull() || m_mediaRecorder.isUndefined()) { + qWarning() << "MediaRecorder could not be found"; + return; + } + m_mediaRecorder.set("data-mediarecordercontext", + emscripten::val(quintptr(reinterpret_cast<void *>(this)))); + + if (!m_mediaStreamDataAvailable.isNull()) { + m_mediaStreamDataAvailable.reset(); + m_mediaStreamStopped.reset(); + m_mediaStreamError.reset(); + m_mediaStreamStart.reset(); + m_mediaStreamPause.reset(); + m_mediaStreamResume.reset(); + } + + // dataavailable + auto callback = [](emscripten::val blob) { + if (blob.isUndefined() || blob.isNull()) { + qCDebug(qWasmMediaRecorder) << "blob is null"; + return; + } + if (blob["target"].isUndefined() || blob["target"].isNull()) + return; + if (blob["data"].isUndefined() || blob["data"].isNull()) + return; + if (blob["target"]["data-mediarecordercontext"].isUndefined() + || blob["target"]["data-mediarecordercontext"].isNull()) + return; + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + blob["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + const double timeCode = + blob.hasOwnProperty("timecode") ? blob["timecode"].as<double>() : 0; + recorder->audioDataAvailable(blob["data"], timeCode); + } + }; + + m_mediaStreamDataAvailable.reset( + new qstdweb::EventCallback(m_mediaRecorder, "dataavailable", callback)); + + // stopped + auto stoppedCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + qCDebug(qWasmMediaRecorder) + << "STOPPED: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = false; + recorder->m_durationTimer->invalidate(); + + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamStopped.reset( + new qstdweb::EventCallback(m_mediaRecorder, "stop", stoppedCallback)); + + // error + auto errorCallback = [](emscripten::val theError) { + if (theError.isUndefined() || theError.isNull()) { + qCDebug(qWasmMediaRecorder) << "error is null"; + return; + } + qCDebug(qWasmMediaRecorder) + << theError["code"].as<int>() + << QString::fromStdString(theError["message"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + theError["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->updateError(QMediaRecorder::ResourceError, + QString::fromStdString(theError["message"].as<std::string>())); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamError.reset(new qstdweb::EventCallback(m_mediaRecorder, "error", errorCallback)); + + // start + auto startCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + + qCDebug(qWasmMediaRecorder) + << "START: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamStart.reset(new qstdweb::EventCallback(m_mediaRecorder, "start", startCallback)); + + // pause + auto pauseCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + + qCDebug(qWasmMediaRecorder) + << "pause: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamPause.reset(new qstdweb::EventCallback(m_mediaRecorder, "pause", pauseCallback)); + + // resume + auto resumeCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + + qCDebug(qWasmMediaRecorder) + << "resume: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamResume.reset( + new qstdweb::EventCallback(m_mediaRecorder, "resume", resumeCallback)); + + // set up what options we can + if (hasCamera()) + setTrackContraints(m_mediaSettings, stream); + else + startStream(); +} + +void QWasmMediaRecorder::audioDataAvailable(emscripten::val blob, double timeCodeDifference) +{ + Q_UNUSED(timeCodeDifference) + if (blob.isUndefined() || blob.isNull()) { + qCDebug(qWasmMediaRecorder) << "blob is null"; + return; + } + + auto fileReader = std::make_shared<qstdweb::FileReader>(); + + fileReader->onError([=](emscripten::val theError) { + updateError(QMediaRecorder::ResourceError, + QString::fromStdString(theError["message"].as<std::string>())); + }); + + fileReader->onAbort([=](emscripten::val) { + updateError(QMediaRecorder::ResourceError, QStringLiteral("File read aborted")); + }); + + fileReader->onLoad([=](emscripten::val) { + if (fileReader->val().isNull() || fileReader->val().isUndefined()) + return; + qstdweb::ArrayBuffer result = fileReader->result(); + if (result.val().isNull() || result.val().isUndefined()) + return; + QByteArray fileContent = qstdweb::Uint8Array(result).copyToQByteArray(); + + if (m_isRecording && !fileContent.isEmpty()) { + m_durationMs = m_durationTimer->elapsed(); + if (m_outputTarget->isOpen()) + m_outputTarget->write(fileContent, fileContent.length()); + // we've read everything + emit durationChanged(m_durationMs); + qCDebug(qWasmMediaRecorder) << "duration changed" << m_durationMs; + } + }); + + fileReader->readAsArrayBuffer(qstdweb::Blob(blob)); +} + +// constraints are suggestions, as not all hardware supports all settings +void QWasmMediaRecorder::setTrackContraints(QMediaEncoderSettings &settings, emscripten::val stream) +{ + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << settings.audioSampleRate(); + + if (stream.isUndefined() || stream.isNull()) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaStream"; + return; + } + + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + // check which ones are supported + // emscripten::val allConstraints = mediaDevices.call<emscripten::val>("getSupportedConstraints"); + // browsers only support some settings + + emscripten::val videoParams = emscripten::val::object(); + emscripten::val constraints = emscripten::val::object(); + + if (hasCamera()) { + if (settings.videoFrameRate() > 0) + videoParams.set("frameRate", emscripten::val(settings.videoFrameRate())); + if (settings.videoResolution().height() > 0) + videoParams.set("height", + emscripten::val(settings.videoResolution().height())); // viewportHeight? + if (settings.videoResolution().width() > 0) + videoParams.set("width", emscripten::val(settings.videoResolution().width())); + + constraints.set("video", videoParams); // only video here + } + + emscripten::val audioParams = emscripten::val::object(); + if (settings.audioSampleRate() > 0) + audioParams.set("sampleRate", emscripten::val(settings.audioSampleRate())); // may not work + if (settings.audioBitRate() > 0) + audioParams.set("sampleSize", emscripten::val(settings.audioBitRate())); // may not work + if (settings.audioChannelCount() > 0) + audioParams.set("channelCount", emscripten::val(settings.audioChannelCount())); + + constraints.set("audio", audioParams); // only audio here + + if (hasCamera() && stream["active"].as<bool>()) { + emscripten::val videoTracks = emscripten::val::undefined(); + videoTracks = stream.call<emscripten::val>("getVideoTracks"); + if (videoTracks.isNull() || videoTracks.isUndefined()) { + qCDebug(qWasmMediaRecorder) << "no video tracks"; + return; + } + if (videoTracks["length"].as<int>() > 0) { + // try to apply the video options + qstdweb::Promise::make(videoTracks[0], QStringLiteral("applyConstraints"), + { .thenFunc = + [this](emscripten::val result) { + Q_UNUSED(result) + startStream(); + }, + .catchFunc = + [this](emscripten::val theError) { + qWarning() << "setting video params failed error"; + qCDebug(qWasmMediaRecorder) + << theError["code"].as<int>() + << QString::fromStdString(theError["message"].as<std::string>()); + updateError(QMediaRecorder::ResourceError, + QString::fromStdString(theError["message"].as<std::string>())); + } }, + constraints); + } + } +} + +// this starts the recording stream +void QWasmMediaRecorder::startStream() +{ + if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "could not find MediaStream"; + return; + } + qCDebug(qWasmMediaRecorder) << "m_mediaRecorder state:" << + QString::fromStdString(m_mediaRecorder["state"].as<std::string>()); + + constexpr int sliceSizeInMs = 250; // TODO find what size is best + m_mediaRecorder.call<void>("start", emscripten::val(sliceSizeInMs)); + + /* this method can optionally be passed a timeslice argument with a value in milliseconds. + * If this is specified, the media will be captured in separate chunks of that duration, + * rather than the default behavior of recording the media in a single large chunk.*/ + + emit stateChanged(state()); +} + +void QWasmMediaRecorder::setUpFileSink() +{ + QString m_targetFileName = outputLocation().toLocalFile(); + QString suffix = m_mediaSettings.mimeType().preferredSuffix(); + if (m_targetFileName.isEmpty()) { + m_targetFileName = "/home/web_user/tmp." + suffix; + QPlatformMediaRecorder::setOutputLocation(m_targetFileName); + } + + m_outputTarget = new QFile(m_targetFileName, this); + if (!m_outputTarget->open(QIODevice::WriteOnly)) { + qWarning() << "target file is not writable"; + return; + } +} + +QT_END_NAMESPACE |