1 files changed, 419 insertions, 0 deletions
diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp
new file mode 100644
index 000000000..4ec10ca84
--- /dev/null
+++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp
@@ -0,0 +1,419 @@
+// Copyright (C) 2016 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include <mediacapture/qgstreamermediaencoder_p.h>
+#include <qgstreamerformatinfo_p.h>
+#include <common/qgstpipeline_p.h>
+#include <common/qgstreamermessage_p.h>
+#include <common/qgst_debug_p.h>
+#include <qgstreamerintegration_p.h>
+
+#include <QtMultimedia/private/qmediastoragelocation_p.h>
+#include <QtMultimedia/private/qplatformcamera_p.h>
+#include <QtMultimedia/qaudiodevice.h>
+
+#include <QtCore/qdebug.h>
+#include <QtCore/qeventloop.h>
+#include <QtCore/qstandardpaths.h>
+#include <QtCore/qmimetype.h>
+#include <QtCore/qloggingcategory.h>
+
+#include <gst/gsttagsetter.h>
+#include <gst/gstversion.h>
+#include <gst/video/video.h>
+#include <gst/pbutils/encoding-profile.h>
+
+static Q_LOGGING_CATEGORY(qLcMediaEncoderGst, "qt.multimedia.encoder")
+
+QT_BEGIN_NAMESPACE
+
+QGstreamerMediaEncoder::QGstreamerMediaEncoder(QMediaRecorder *parent)
+  : QPlatformMediaRecorder(parent),
+    audioPauseControl(*this),
+    videoPauseControl(*this)
+{
+    signalDurationChangedTimer.setInterval(100);
+    signalDurationChangedTimer.callOnTimeout(&signalDurationChangedTimer, [this]() {
+        durationChanged(duration());
+    });
+}
+
+QGstreamerMediaEncoder::~QGstreamerMediaEncoder()
+{
+    if (!capturePipeline.isNull()) {
+        finalize();
+        capturePipeline.removeMessageFilter(this);
+        capturePipeline.setStateSync(GST_STATE_NULL);
+    }
+}
+
+bool QGstreamerMediaEncoder::isLocationWritable(const QUrl &) const
+{
+    return true;
+}
+
+void QGstreamerMediaEncoder::handleSessionError(QMediaRecorder::Error code, const QString &description)
+{
+    updateError(code, description);
+    stop();
+}
+
+bool QGstreamerMediaEncoder::processBusMessage(const QGstreamerMessage &msg)
+{
+    constexpr bool traceStateChange = false;
+    constexpr bool traceAllEvents = false;
+
+    if constexpr (traceAllEvents)
+        qCDebug(qLcMediaEncoderGst) << "received event:" << msg;
+
+    switch (msg.type()) {
+    case GST_MESSAGE_ELEMENT: {
+        QGstStructureView s = msg.structure();
+        if (s.name() == "GstBinForwarded")
+            return processBusMessage(s.getMessage());
+
+        qCDebug(qLcMediaEncoderGst)
+                << "received element message from" << msg.source().name() << s.name();
+        return false;
+    }
+
+    case GST_MESSAGE_EOS: {
+        qCDebug(qLcMediaEncoderGst) << "received EOS from" << msg.source().name();
+        finalize();
+        return false;
+    }
+
+    case GST_MESSAGE_ERROR: {
+        qCDebug(qLcMediaEncoderGst)
+                << "received error:" << msg.source().name() << QCompactGstMessageAdaptor(msg);
+
+        QUniqueGErrorHandle err;
+        QGString debug;
+        gst_message_parse_error(msg.message(), &err, &debug);
+        updateError(QMediaRecorder::ResourceError, QString::fromUtf8(err.get()->message));
+        if (!m_finalizing)
+            stop();
+        finalize();
+        return false;
+    }
+
+    case GST_MESSAGE_STATE_CHANGED: {
+        if constexpr (traceStateChange)
+            qCDebug(qLcMediaEncoderGst)
+                    << "received state change" << QCompactGstMessageAdaptor(msg);
+
+        return false;
+    }
+
+    default:
+        return false;
+    };
+}
+
+qint64 QGstreamerMediaEncoder::duration() const
+{
+    return std::max(audioPauseControl.duration, videoPauseControl.duration);
+}
+
+
+static GstEncodingContainerProfile *createContainerProfile(const QMediaEncoderSettings &settings)
+{
+    auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
+
+    auto caps = formatInfo->formatCaps(settings.fileFormat());
+
+    GstEncodingContainerProfile *profile =
+            (GstEncodingContainerProfile *)gst_encoding_container_profile_new(
+                    "container_profile", (gchar *)"custom container profile",
+                    const_cast<GstCaps *>(caps.caps()),
+                    nullptr); // preset
+    return profile;
+}
+
+static GstEncodingProfile *createVideoProfile(const QMediaEncoderSettings &settings)
+{
+    auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
+
+    QGstCaps caps = formatInfo->videoCaps(settings.mediaFormat());
+    if (caps.isNull())
+        return nullptr;
+
+    QSize videoResolution = settings.videoResolution();
+    if (videoResolution.isValid())
+        caps.setResolution(videoResolution);
+
+    GstEncodingVideoProfile *profile =
+            gst_encoding_video_profile_new(const_cast<GstCaps *>(caps.caps()), nullptr,
+                                           nullptr, // restriction
+                                           0); // presence
+
+    gst_encoding_video_profile_set_pass(profile, 0);
+    gst_encoding_video_profile_set_variableframerate(profile, TRUE);
+
+    return (GstEncodingProfile *)profile;
+}
+
+static GstEncodingProfile *createAudioProfile(const QMediaEncoderSettings &settings)
+{
+    auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
+
+    auto caps = formatInfo->audioCaps(settings.mediaFormat());
+    if (caps.isNull())
+        return nullptr;
+
+    GstEncodingProfile *profile =
+            (GstEncodingProfile *)gst_encoding_audio_profile_new(const_cast<GstCaps *>(caps.caps()),
+                                                                 nullptr, // preset
+                                                                 nullptr, // restriction
+                                                                 0); // presence
+
+    return profile;
+}
+
+
+static GstEncodingContainerProfile *createEncodingProfile(const QMediaEncoderSettings &settings)
+{
+    auto *containerProfile = createContainerProfile(settings);
+    if (!containerProfile) {
+        qWarning() << "QGstreamerMediaEncoder: failed to create container profile!";
+        return nullptr;
+    }
+
+    GstEncodingProfile *audioProfile = createAudioProfile(settings);
+    GstEncodingProfile *videoProfile = nullptr;
+    if (settings.videoCodec() != QMediaFormat::VideoCodec::Unspecified)
+        videoProfile = createVideoProfile(settings);
+//    qDebug() << "audio profile" << (audioProfile ? gst_caps_to_string(gst_encoding_profile_get_format(audioProfile)) : "(null)");
+//    qDebug() << "video profile" << (videoProfile ? gst_caps_to_string(gst_encoding_profile_get_format(videoProfile)) : "(null)");
+//    qDebug() << "conta profile" << gst_caps_to_string(gst_encoding_profile_get_format((GstEncodingProfile *)containerProfile));
+
+    if (videoProfile) {
+        if (!gst_encoding_container_profile_add_profile(containerProfile, videoProfile)) {
+            qWarning() << "QGstreamerMediaEncoder: failed to add video profile!";
+            gst_encoding_profile_unref(videoProfile);
+        }
+    }
+    if (audioProfile) {
+        if (!gst_encoding_container_profile_add_profile(containerProfile, audioProfile)) {
+            qWarning() << "QGstreamerMediaEncoder: failed to add audio profile!";
+            gst_encoding_profile_unref(audioProfile);
+        }
+    }
+
+    return containerProfile;
+}
+
+void QGstreamerMediaEncoder::PauseControl::reset()
+{
+    pauseOffsetPts = 0;
+    pauseStartPts.reset();
+    duration = 0;
+    firstBufferPts.reset();
+}
+
+void QGstreamerMediaEncoder::PauseControl::installOn(QGstPad pad)
+{
+    pad.addProbe<&QGstreamerMediaEncoder::PauseControl::processBuffer>(this, GST_PAD_PROBE_TYPE_BUFFER);
+}
+
+GstPadProbeReturn QGstreamerMediaEncoder::PauseControl::processBuffer(QGstPad, GstPadProbeInfo *info)
+{
+    auto buffer = GST_PAD_PROBE_INFO_BUFFER(info);
+    if (!buffer)
+        return GST_PAD_PROBE_OK;
+
+    buffer = gst_buffer_make_writable(buffer);
+
+    if (!buffer)
+        return GST_PAD_PROBE_OK;
+
+    GST_PAD_PROBE_INFO_DATA(info) = buffer;
+
+    if (!GST_BUFFER_PTS_IS_VALID(buffer))
+        return GST_PAD_PROBE_OK;
+
+    if (!firstBufferPts)
+        firstBufferPts = GST_BUFFER_PTS(buffer);
+
+    if (encoder.state() == QMediaRecorder::PausedState) {
+        if (!pauseStartPts)
+            pauseStartPts = GST_BUFFER_PTS(buffer);
+
+        return GST_PAD_PROBE_DROP;
+    }
+
+    if (pauseStartPts) {
+        pauseOffsetPts += GST_BUFFER_PTS(buffer) - *pauseStartPts;
+        pauseStartPts.reset();
+    }
+    GST_BUFFER_PTS(buffer) -= pauseOffsetPts;
+
+    duration = (GST_BUFFER_PTS(buffer) - *firstBufferPts) / GST_MSECOND;
+
+    return GST_PAD_PROBE_OK;
+}
+
+void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings)
+{
+    if (!m_session ||m_finalizing || state() != QMediaRecorder::StoppedState)
+        return;
+
+    const auto hasVideo = m_session->camera() && m_session->camera()->isActive();
+    const auto hasAudio = m_session->audioInput() != nullptr;
+
+    if (!hasVideo && !hasAudio) {
+        updateError(QMediaRecorder::ResourceError, QMediaRecorder::tr("No camera or audio input"));
+        return;
+    }
+
+    const auto audioOnly = settings.videoCodec() == QMediaFormat::VideoCodec::Unspecified;
+
+    auto primaryLocation = audioOnly ? QStandardPaths::MusicLocation : QStandardPaths::MoviesLocation;
+    auto container = settings.mimeType().preferredSuffix();
+    auto location = QMediaStorageLocation::generateFileName(outputLocation().toLocalFile(), primaryLocation, container);
+
+    QUrl actualSink = QUrl::fromLocalFile(QDir::currentPath()).resolved(location);
+    qCDebug(qLcMediaEncoderGst) << "recording new video to" << actualSink;
+
+    Q_ASSERT(!actualSink.isEmpty());
+
+    gstEncoder = QGstBin::createFromFactory("encodebin", "encodebin");
+    Q_ASSERT(gstEncoder);
+    auto *encodingProfile = createEncodingProfile(settings);
+    g_object_set (gstEncoder.object(), "profile", encodingProfile, nullptr);
+    gst_encoding_profile_unref(encodingProfile);
+
+    gstFileSink = QGstElement::createFromFactory("filesink", "filesink");
+    Q_ASSERT(gstFileSink);
+    gstFileSink.set("location", QFile::encodeName(actualSink.toLocalFile()).constData());
+    gstFileSink.set("async", false);
+
+    QGstPad audioSink = {};
+    QGstPad videoSink = {};
+
+    audioPauseControl.reset();
+    videoPauseControl.reset();
+
+    if (hasAudio) {
+        audioSink = gstEncoder.getRequestPad("audio_%u");
+        if (audioSink.isNull())
+            qWarning() << "Unsupported audio codec";
+        else
+            audioPauseControl.installOn(audioSink);
+    }
+
+    if (hasVideo) {
+        videoSink = gstEncoder.getRequestPad("video_%u");
+        if (videoSink.isNull())
+            qWarning() << "Unsupported video codec";
+        else
+            videoPauseControl.installOn(videoSink);
+    }
+
+    capturePipeline.modifyPipelineWhileNotRunning([&] {
+        capturePipeline.add(gstEncoder, gstFileSink);
+        qLinkGstElements(gstEncoder, gstFileSink);
+        applyMetaDataToTagSetter(m_metaData, gstEncoder);
+
+        m_session->linkEncoder(audioSink, videoSink);
+
+        gstEncoder.syncStateWithParent();
+        gstFileSink.syncStateWithParent();
+    });
+
+    signalDurationChangedTimer.start();
+    capturePipeline.dumpGraph("recording");
+
+    durationChanged(0);
+    stateChanged(QMediaRecorder::RecordingState);
+    actualLocationChanged(QUrl::fromLocalFile(location));
+}
+
+void QGstreamerMediaEncoder::pause()
+{
+    if (!m_session || m_finalizing || state() != QMediaRecorder::RecordingState)
+        return;
+    signalDurationChangedTimer.stop();
+    durationChanged(duration());
+    capturePipeline.dumpGraph("before-pause");
+    stateChanged(QMediaRecorder::PausedState);
+}
+
+void QGstreamerMediaEncoder::resume()
+{
+    capturePipeline.dumpGraph("before-resume");
+    if (!m_session || m_finalizing || state() != QMediaRecorder::PausedState)
+        return;
+    signalDurationChangedTimer.start();
+    stateChanged(QMediaRecorder::RecordingState);
+}
+
+void QGstreamerMediaEncoder::stop()
+{
+    if (!m_session || m_finalizing || state() == QMediaRecorder::StoppedState)
+        return;
+    durationChanged(duration());
+    qCDebug(qLcMediaEncoderGst) << "stop";
+    m_finalizing = true;
+    m_session->unlinkEncoder();
+    signalDurationChangedTimer.stop();
+
+    qCDebug(qLcMediaEncoderGst) << ">>>>>>>>>>>>> sending EOS";
+    gstEncoder.sendEos();
+}
+
+void QGstreamerMediaEncoder::finalize()
+{
+    if (!m_session || gstEncoder.isNull())
+        return;
+
+    qCDebug(qLcMediaEncoderGst) << "finalize";
+
+    capturePipeline.stopAndRemoveElements(gstEncoder, gstFileSink);
+    gstFileSink = {};
+    gstEncoder = {};
+    m_finalizing = false;
+    stateChanged(QMediaRecorder::StoppedState);
+}
+
+void QGstreamerMediaEncoder::setMetaData(const QMediaMetaData &metaData)
+{
+    if (!m_session)
+        return;
+    m_metaData = metaData;
+}
+
+QMediaMetaData QGstreamerMediaEncoder::metaData() const
+{
+    return m_metaData;
+}
+
+void QGstreamerMediaEncoder::setCaptureSession(QPlatformMediaCaptureSession *session)
+{
+    QGstreamerMediaCapture *captureSession = static_cast<QGstreamerMediaCapture *>(session);
+    if (m_session == captureSession)
+        return;
+
+    if (m_session) {
+        stop();
+        if (m_finalizing) {
+            QEventLoop loop;
+            QObject::connect(mediaRecorder(), &QMediaRecorder::recorderStateChanged, &loop,
+                             &QEventLoop::quit);
+            loop.exec();
+        }
+
+        capturePipeline.removeMessageFilter(this);
+        capturePipeline = {};
+    }
+
+    m_session = captureSession;
+    if (!m_session)
+        return;
+
+    capturePipeline = captureSession->capturePipeline;
+    capturePipeline.set("message-forward", true);
+    capturePipeline.installMessageFilter(this);
+}
+
+QT_END_NAMESPACE