diff options
Diffstat (limited to 'src/plugins/multimedia/gstreamer')
62 files changed, 7987 insertions, 7227 deletions
diff --git a/src/plugins/multimedia/gstreamer/CMakeLists.txt b/src/plugins/multimedia/gstreamer/CMakeLists.txt index 3bce143f6..1ef1f9a36 100644 --- a/src/plugins/multimedia/gstreamer/CMakeLists.txt +++ b/src/plugins/multimedia/gstreamer/CMakeLists.txt @@ -1,20 +1,24 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_find_package(EGL) -qt_internal_add_plugin(QGstreamerMediaPlugin - OUTPUT_NAME gstreamermediaplugin - PLUGIN_TYPE multimedia +qt_internal_add_module(QGstreamerMediaPluginPrivate + STATIC + INTERNAL_MODULE SOURCES audio/qgstreameraudiodevice.cpp audio/qgstreameraudiodevice_p.h - audio/qgstreameraudiosource.cpp audio/qgstreameraudiosource_p.h - audio/qgstreameraudiosink.cpp audio/qgstreameraudiosink_p.h audio/qgstreameraudiodecoder.cpp audio/qgstreameraudiodecoder_p.h - common/qgst_p.h - common/qgstappsrc.cpp common/qgstappsrc_p.h + common/qglist_helper_p.h + common/qgst.cpp common/qgst_p.h + common/qgst_debug.cpp common/qgst_debug_p.h + common/qgst_handle_types_p.h + common/qgstappsource.cpp common/qgstappsource_p.h common/qgstreameraudioinput.cpp common/qgstreameraudioinput_p.h common/qgstreameraudiooutput.cpp common/qgstreameraudiooutput_p.h common/qgstreamerbufferprobe.cpp common/qgstreamerbufferprobe_p.h common/qgstreamermetadata.cpp common/qgstreamermetadata_p.h - common/qgstreamermessage.cpp common/qgstreamermessage_p.h + common/qgstreamermessage_p.h common/qgstreamermediaplayer.cpp common/qgstreamermediaplayer_p.h common/qgstreamervideooutput.cpp common/qgstreamervideooutput_p.h common/qgstreamervideooverlay.cpp common/qgstreamervideooverlay_p.h @@ -24,33 +28,46 @@ qt_internal_add_plugin(QGstreamerMediaPlugin common/qgstvideobuffer.cpp common/qgstvideobuffer_p.h common/qgstvideorenderersink.cpp common/qgstvideorenderersink_p.h common/qgstsubtitlesink.cpp common/qgstsubtitlesink_p.h - qgstreamervideodevices.cpp qgstreamervideodevices_p.h - qgstreamerformatinfo.cpp qgstreamerformatinfo_p.h qgstreamerintegration.cpp qgstreamerintegration_p.h + qgstreamerformatinfo.cpp qgstreamerformatinfo_p.h + qgstreamervideodevices.cpp qgstreamervideodevices_p.h mediacapture/qgstreamercamera.cpp mediacapture/qgstreamercamera_p.h mediacapture/qgstreamerimagecapture.cpp mediacapture/qgstreamerimagecapture_p.h mediacapture/qgstreamermediacapture.cpp mediacapture/qgstreamermediacapture_p.h mediacapture/qgstreamermediaencoder.cpp mediacapture/qgstreamermediaencoder_p.h + NO_UNITY_BUILD_SOURCES + # Conflicts with macros defined in X11.h, and Xlib.h + common/qgstvideobuffer.cpp + common/qgstreamervideosink.cpp + NO_GENERATE_CPP_EXPORTS DEFINES GLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_26 - INCLUDE_DIRECTORIES - audio - common - mediacapture - LIBRARIES + PUBLIC_LIBRARIES Qt::MultimediaPrivate Qt::CorePrivate GStreamer::GStreamer GStreamer::App ) -qt_internal_extend_target(QGstreamerMediaPlugin CONDITION QT_FEATURE_gstreamer_photography - LIBRARIES - -lgstphotography-1.0 +qt_internal_extend_target(QGstreamerMediaPluginPrivate CONDITION QT_FEATURE_gstreamer_photography + PUBLIC_LIBRARIES + GStreamer::Photography ) -qt_internal_extend_target(QGstreamerMediaPlugin CONDITION QT_FEATURE_gstreamer_gl - LIBRARIES +qt_internal_extend_target(QGstreamerMediaPluginPrivate CONDITION QT_FEATURE_gstreamer_gl + PUBLIC_LIBRARIES GStreamer::Gl + LIBRARIES EGL::EGL ) + +qt_internal_add_plugin(QGstreamerMediaPlugin + OUTPUT_NAME gstreamermediaplugin + PLUGIN_TYPE multimedia + SOURCES + qgstreamerplugin.cpp + gstreamer.json + LIBRARIES + Qt::QGstreamerMediaPluginPrivate + Qt::MultimediaPrivate +) diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp index 779084c7f..280b43cdb 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp @@ -1,47 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only //#define DEBUG_DECODER -#include "qgstreameraudiodecoder_p.h" -#include "qgstreamermessage_p.h" +#include <audio/qgstreameraudiodecoder_p.h> -#include <qgstutils_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstutils_p.h> #include <gst/gstvalue.h> #include <gst/base/gstbasesrc.h> @@ -54,11 +19,12 @@ #include <QtCore/qdir.h> #include <QtCore/qstandardpaths.h> #include <QtCore/qurl.h> - -#define MAX_BUFFERS_IN_QUEUE 4 +#include <QtCore/qloggingcategory.h> QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLcGstreamerAudioDecoder, "qt.multimedia.gstreameraudiodecoder"); + typedef enum { GST_PLAY_FLAG_VIDEO = 0x00000001, GST_PLAY_FLAG_AUDIO = 0x00000002, @@ -72,32 +38,41 @@ typedef enum { } GstPlayFlags; +QMaybe<QPlatformAudioDecoder *> QGstreamerAudioDecoder::create(QAudioDecoder *parent) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable("audioconvert", "playbin"); + if (error) + return *error; + + return new QGstreamerAudioDecoder(parent); +} QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent) : QPlatformAudioDecoder(parent), - m_playbin(GST_PIPELINE_CAST(QGstElement("playbin", "playbin").element())) + m_playbin{ + QGstPipeline::adopt(GST_PIPELINE_CAST( + QGstElement::createFromFactory("playbin", "playbin").element())), + }, + m_audioConvert{ + QGstElement::createFromFactory("audioconvert", "audioconvert"), + } { - if (m_playbin.isNull()) { - // ### set error - return; - } - // Sort out messages m_playbin.installMessageFilter(this); // Set the rest of the pipeline up setAudioFlags(true); - m_audioConvert = QGstElement("audioconvert", "audioconvert"); - - m_outputBin = QGstBin("audio-output-bin"); + m_outputBin = QGstBin::create("audio-output-bin"); m_outputBin.add(m_audioConvert); // add ghostpad m_outputBin.addGhostPad(m_audioConvert, "sink"); g_object_set(m_playbin.object(), "audio-sink", m_outputBin.element(), NULL); - g_signal_connect(m_playbin.object(), "deep-notify::source", (GCallback) &QGstreamerAudioDecoder::configureAppSrcElement, (gpointer)this); + + m_deepNotifySourceConnection = m_playbin.connect( + "deep-notify::source", (GCallback)&configureAppSrcElement, (gpointer)this); // Set volume to 100% gdouble volume = 1.0; @@ -106,165 +81,156 @@ QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent) QGstreamerAudioDecoder::~QGstreamerAudioDecoder() { - if (m_playbin.isNull()) - return; - stop(); -#if QT_CONFIG(gstreamer_app) + m_playbin.removeMessageFilter(this); + delete m_appSrc; -#endif } -#if QT_CONFIG(gstreamer_app) -void QGstreamerAudioDecoder::configureAppSrcElement(GObject* object, GObject *orig, GParamSpec *pspec, QGstreamerAudioDecoder *self) +void QGstreamerAudioDecoder::configureAppSrcElement([[maybe_unused]] GObject *object, GObject *orig, + [[maybe_unused]] GParamSpec *pspec, + QGstreamerAudioDecoder *self) { - Q_UNUSED(object); - Q_UNUSED(pspec); - // In case we switch from appsrc to not - if (!self->appsrc()) + if (!self->m_appSrc) return; - GstElement *appsrc; + QGstElementHandle appsrc; g_object_get(orig, "source", &appsrc, NULL); - auto *qAppSrc = self->appsrc(); - qAppSrc->setExternalAppSrc(appsrc); + auto *qAppSrc = self->m_appSrc; + qAppSrc->setExternalAppSrc(QGstAppSrc{ + qGstSafeCast<GstAppSrc>(appsrc.get()), + QGstAppSrc::NeedsRef, // CHECK: can we `release()`? + }); qAppSrc->setup(self->mDevice); - - g_object_unref(G_OBJECT(appsrc)); } -#endif bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message) { - GstMessage* gm = message.rawMessage(); - if (gm) { - if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_DURATION) { - updateDuration(); - } else if (GST_MESSAGE_SRC(gm) == m_playbin.object()) { - switch (GST_MESSAGE_TYPE(gm)) { - case GST_MESSAGE_STATE_CHANGED: - { - GstState oldState; - GstState newState; - GstState pending; - - gst_message_parse_state_changed(gm, &oldState, &newState, &pending); - -#ifdef DEBUG_DECODER - QStringList states; - states << "GST_STATE_VOID_PENDING" << "GST_STATE_NULL" << "GST_STATE_READY" << "GST_STATE_PAUSED" << "GST_STATE_PLAYING"; - - qDebug() << QString("state changed: old: %1 new: %2 pending: %3") \ - .arg(states[oldState]) \ - .arg(states[newState]) \ - .arg(states[pending]) << "internal" << m_state; -#endif + qCDebug(qLcGstreamerAudioDecoder) << "received bus message:" << message; - bool isDecoding = false; - switch (newState) { - case GST_STATE_VOID_PENDING: - case GST_STATE_NULL: - case GST_STATE_READY: - break; - case GST_STATE_PLAYING: - isDecoding = true; - break; - case GST_STATE_PAUSED: - isDecoding = true; - - //gstreamer doesn't give a reliable indication the duration - //information is ready, GST_MESSAGE_DURATION is not sent by most elements - //the duration is queried up to 5 times with increasing delay - m_durationQueries = 5; - updateDuration(); - break; - } - - setIsDecoding(isDecoding); - } - break; - - case GST_MESSAGE_EOS: - finished(); - break; - - case GST_MESSAGE_ERROR: { - GError *err; - gchar *debug; - gst_message_parse_error(gm, &err, &debug); - if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) - processInvalidMedia(QAudioDecoder::FormatError, tr("Cannot play stream of type: <unknown>")); - else - processInvalidMedia(QAudioDecoder::ResourceError, QString::fromUtf8(err->message)); - qWarning() << "Error:" << QString::fromUtf8(err->message); - g_error_free(err); - g_free(debug); - } - break; - case GST_MESSAGE_WARNING: - { - GError *err; - gchar *debug; - gst_message_parse_warning (gm, &err, &debug); - qWarning() << "Warning:" << QString::fromUtf8(err->message); - g_error_free (err); - g_free (debug); - } - break; -#ifdef DEBUG_DECODER - case GST_MESSAGE_INFO: - { - GError *err; - gchar *debug; - gst_message_parse_info (gm, &err, &debug); - qDebug() << "Info:" << QString::fromUtf8(err->message); - g_error_free (err); - g_free (debug); - } - break; -#endif - default: - break; - } - } else if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_ERROR) { - GError *err; - gchar *debug; - gst_message_parse_error(gm, &err, &debug); + GstMessage *gm = message.message(); + + switch (message.type()) { + case GST_MESSAGE_DURATION: { + updateDuration(); + return false; + } + + case GST_MESSAGE_ERROR: { + qCDebug(qLcGstreamerAudioDecoder) << " error" << QCompactGstMessageAdaptor(message); + + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(gm, &err, &debug); + + if (message.source() == m_playbin) { + if (err.get()->domain == GST_STREAM_ERROR + && err.get()->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) + processInvalidMedia(QAudioDecoder::FormatError, + tr("Cannot play stream of type: <unknown>")); + else + processInvalidMedia(QAudioDecoder::ResourceError, + QString::fromUtf8(err.get()->message)); + } else { QAudioDecoder::Error qerror = QAudioDecoder::ResourceError; - if (err->domain == GST_STREAM_ERROR) { - switch (err->code) { - case GST_STREAM_ERROR_DECRYPT: - case GST_STREAM_ERROR_DECRYPT_NOKEY: - qerror = QAudioDecoder::AccessDeniedError; - break; - case GST_STREAM_ERROR_FORMAT: - case GST_STREAM_ERROR_DEMUX: - case GST_STREAM_ERROR_DECODE: - case GST_STREAM_ERROR_WRONG_TYPE: - case GST_STREAM_ERROR_TYPE_NOT_FOUND: - case GST_STREAM_ERROR_CODEC_NOT_FOUND: - qerror = QAudioDecoder::FormatError; - break; - default: - break; + if (err.get()->domain == GST_STREAM_ERROR) { + switch (err.get()->code) { + case GST_STREAM_ERROR_DECRYPT: + case GST_STREAM_ERROR_DECRYPT_NOKEY: + qerror = QAudioDecoder::AccessDeniedError; + break; + case GST_STREAM_ERROR_FORMAT: + case GST_STREAM_ERROR_DEMUX: + case GST_STREAM_ERROR_DECODE: + case GST_STREAM_ERROR_WRONG_TYPE: + case GST_STREAM_ERROR_TYPE_NOT_FOUND: + case GST_STREAM_ERROR_CODEC_NOT_FOUND: + qerror = QAudioDecoder::FormatError; + break; + default: + break; } - } else if (err->domain == GST_CORE_ERROR) { - switch (err->code) { - case GST_CORE_ERROR_MISSING_PLUGIN: - qerror = QAudioDecoder::FormatError; - break; - default: - break; + } else if (err.get()->domain == GST_CORE_ERROR) { + switch (err.get()->code) { + case GST_CORE_ERROR_MISSING_PLUGIN: + qerror = QAudioDecoder::FormatError; + break; + default: + break; } } - processInvalidMedia(qerror, QString::fromUtf8(err->message)); - g_error_free(err); - g_free(debug); + processInvalidMedia(qerror, QString::fromUtf8(err.get()->message)); } + break; + } + + default: + if (message.source() == m_playbin) + return handlePlaybinMessage(message); + } + + return false; +} + +bool QGstreamerAudioDecoder::handlePlaybinMessage(const QGstreamerMessage &message) +{ + GstMessage *gm = message.message(); + + switch (GST_MESSAGE_TYPE(gm)) { + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(gm, &oldState, &newState, &pending); + + bool isDecoding = false; + switch (newState) { + case GST_STATE_VOID_PENDING: + case GST_STATE_NULL: + case GST_STATE_READY: + break; + case GST_STATE_PLAYING: + isDecoding = true; + break; + case GST_STATE_PAUSED: + isDecoding = true; + + // gstreamer doesn't give a reliable indication the duration + // information is ready, GST_MESSAGE_DURATION is not sent by most elements + // the duration is queried up to 5 times with increasing delay + m_durationQueries = 5; + updateDuration(); + break; + } + + setIsDecoding(isDecoding); + break; + }; + + case GST_MESSAGE_EOS: + m_playbin.setState(GST_STATE_NULL); + finished(); + break; + + case GST_MESSAGE_ERROR: + Q_UNREACHABLE_RETURN(false); // handled in processBusMessage + + case GST_MESSAGE_WARNING: + qCWarning(qLcGstreamerAudioDecoder) << "Warning:" << QCompactGstMessageAdaptor(message); + break; + + case GST_MESSAGE_INFO: { + if (qLcGstreamerAudioDecoder().isDebugEnabled()) + qCWarning(qLcGstreamerAudioDecoder) << "Info:" << QCompactGstMessageAdaptor(message); + break; + } + default: + break; } return false; @@ -285,7 +251,7 @@ void QGstreamerAudioDecoder::setSource(const QUrl &fileName) bool isSignalRequired = (mSource != fileName); mSource = fileName; if (isSignalRequired) - emit sourceChanged(); + sourceChanged(); } QIODevice *QGstreamerAudioDecoder::sourceDevice() const @@ -300,16 +266,11 @@ void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device) bool isSignalRequired = (mDevice != device); mDevice = device; if (isSignalRequired) - emit sourceChanged(); + sourceChanged(); } void QGstreamerAudioDecoder::start() { - if (m_playbin.isNull()) { - processInvalidMedia(QAudioDecoder::ResourceError, QLatin1String("Playbin element is not valid")); - return; - } - addAppSink(); if (!mSource.isEmpty()) { @@ -321,8 +282,15 @@ void QGstreamerAudioDecoder::start() return; } - if (!m_appSrc) - m_appSrc = new QGstAppSrc(this); + if (!m_appSrc) { + auto maybeAppSrc = QGstAppSource::create(this); + if (maybeAppSrc) { + m_appSrc = maybeAppSrc.value(); + } else { + processInvalidMedia(QAudioDecoder::ResourceError, maybeAppSrc.error()); + return; + } + } m_playbin.set("uri", "appsrc://"); } else { @@ -333,12 +301,12 @@ void QGstreamerAudioDecoder::start() if (m_appSink) { if (mFormat.isValid()) { setAudioFlags(false); - QGstMutableCaps caps = QGstUtils::capsForAudioFormat(mFormat); - gst_app_sink_set_caps(m_appSink, caps.get()); + auto caps = QGstUtils::capsForAudioFormat(mFormat); + m_appSink.setCaps(caps); } else { // We want whatever the native audio format is setAudioFlags(true); - gst_app_sink_set_caps(m_appSink, nullptr); + m_appSink.setCaps({}); } } @@ -351,26 +319,24 @@ void QGstreamerAudioDecoder::start() void QGstreamerAudioDecoder::stop() { - if (m_playbin.isNull()) - return; - m_playbin.setState(GST_STATE_NULL); + m_currentSessionId += 1; removeAppSink(); // GStreamer thread is stopped. Can safely access m_buffersAvailable if (m_buffersAvailable != 0) { m_buffersAvailable = 0; - emit bufferAvailableChanged(false); + bufferAvailableChanged(false); } - if (m_position != -1) { - m_position = -1; - emit positionChanged(m_position); + if (m_position != invalidPosition) { + m_position = invalidPosition; + positionChanged(m_position.count()); } - if (m_duration != -1) { - m_duration = -1; - emit durationChanged(m_duration); + if (m_duration != invalidDuration) { + m_duration = invalidDuration; + durationChanged(m_duration.count()); } setIsDecoding(false); @@ -385,104 +351,94 @@ void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format) { if (mFormat != format) { mFormat = format; - emit formatChanged(mFormat); + formatChanged(mFormat); } } QAudioBuffer QGstreamerAudioDecoder::read() { - QAudioBuffer audioBuffer; - - int buffersAvailable; - { - QMutexLocker locker(&m_buffersMutex); - buffersAvailable = m_buffersAvailable; - - // need to decrement before pulling a buffer - // to make sure assert in QGstreamerAudioDecoderControl::new_buffer works - m_buffersAvailable--; - } + using namespace std::chrono; + QAudioBuffer audioBuffer; - if (buffersAvailable) { - if (buffersAvailable == 1) - emit bufferAvailableChanged(false); - - const char* bufferData = nullptr; - int bufferSize = 0; - - GstSample *sample = gst_app_sink_pull_sample(m_appSink); - GstBuffer *buffer = gst_sample_get_buffer(sample); - GstMapInfo mapInfo; - gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); - bufferData = (const char*)mapInfo.data; - bufferSize = mapInfo.size; - QAudioFormat format = QGstUtils::audioFormatForSample(sample); - - if (format.isValid()) { - // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. - // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. - qint64 position = getPositionFromBuffer(buffer); - audioBuffer = QAudioBuffer(QByteArray((const char*)bufferData, bufferSize), format, position); - position /= 1000; // convert to milliseconds - if (position != m_position) { - m_position = position; - emit positionChanged(m_position); - } + if (m_buffersAvailable == 0) + return audioBuffer; + + m_buffersAvailable -= 1; + + if (m_buffersAvailable == 0) + bufferAvailableChanged(false); + + QGstSampleHandle sample = m_appSink.pullSample(); + GstBuffer *buffer = gst_sample_get_buffer(sample.get()); + GstMapInfo mapInfo; + gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); + const char *bufferData = (const char *)mapInfo.data; + int bufferSize = mapInfo.size; + QAudioFormat format = QGstUtils::audioFormatForSample(sample.get()); + + if (format.isValid()) { + // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. + // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. + nanoseconds position = getPositionFromBuffer(buffer); + audioBuffer = QAudioBuffer{ + QByteArray(bufferData, bufferSize), + format, + round<microseconds>(position).count(), + }; + milliseconds positionInMs = round<milliseconds>(position); + if (position != m_position) { + m_position = positionInMs; + positionChanged(m_position.count()); } - gst_buffer_unmap(buffer, &mapInfo); - gst_sample_unref(sample); } + gst_buffer_unmap(buffer, &mapInfo); return audioBuffer; } -bool QGstreamerAudioDecoder::bufferAvailable() const -{ - QMutexLocker locker(&m_buffersMutex); - return m_buffersAvailable > 0; -} - qint64 QGstreamerAudioDecoder::position() const { - return m_position; + return m_position.count(); } qint64 QGstreamerAudioDecoder::duration() const { - return m_duration; + return m_duration.count(); } void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString) { stop(); - emit error(int(errorCode), errorString); + error(int(errorCode), errorString); } -GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *, gpointer user_data) +GstFlowReturn QGstreamerAudioDecoder::newSample(GstAppSink *) { - // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()." - QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder*>(user_data); - - int buffersAvailable; - { - QMutexLocker locker(&decoder->m_buffersMutex); - buffersAvailable = decoder->m_buffersAvailable; - decoder->m_buffersAvailable++; - Q_ASSERT(decoder->m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE); - } + // "Note that the preroll buffer will also be returned as the first buffer when calling + // gst_app_sink_pull_buffer()." + + QMetaObject::invokeMethod(this, [this, sessionId = m_currentSessionId] { + if (sessionId != m_currentSessionId) + return; // stop()ed before message is executed + + m_buffersAvailable += 1; + bufferAvailableChanged(true); + bufferReady(); + }); - if (!buffersAvailable) - decoder->bufferAvailableChanged(true); - decoder->bufferReady(); return GST_FLOW_OK; } -void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) +GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *sink, gpointer user_data) { - if (m_playbin.isNull()) - return; + QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder *>(user_data); + qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::new_sample"; + return decoder->newSample(sink); +} +void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) +{ int flags = m_playbin.getInt("flags"); // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired // it prevents audio format conversion @@ -495,20 +451,32 @@ void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) void QGstreamerAudioDecoder::addAppSink() { + using namespace std::chrono_literals; + if (m_appSink) return; - m_appSink = (GstAppSink*)gst_element_factory_make("appsink", nullptr); + qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::addAppSink"; + m_appSink = QGstAppSink::create("decoderAppSink"); + GstAppSinkCallbacks callbacks{}; + callbacks.new_sample = new_sample; + m_appSink.setCallbacks(callbacks, this, nullptr); + +#if GST_CHECK_VERSION(1, 24, 0) + static constexpr auto maxBufferTime = 500ms; + m_appSink.setMaxBufferTime(maxBufferTime); +#else + static constexpr int maxBuffers = 16; + m_appSink.setMaxBuffers(maxBuffers); +#endif - GstAppSinkCallbacks callbacks; - memset(&callbacks, 0, sizeof(callbacks)); - callbacks.new_sample = &new_sample; - gst_app_sink_set_callbacks(m_appSink, &callbacks, this, nullptr); - gst_app_sink_set_max_buffers(m_appSink, MAX_BUFFERS_IN_QUEUE); - gst_base_sink_set_sync(GST_BASE_SINK(m_appSink), FALSE); + static constexpr bool sync = false; + m_appSink.setSync(sync); - gst_bin_add(m_outputBin.bin(), GST_ELEMENT(m_appSink)); - gst_element_link(m_audioConvert.element(), GST_ELEMENT(m_appSink)); + QGstPipeline::modifyPipelineWhileNotRunning(m_playbin.getPipeline(), [&] { + m_outputBin.add(m_appSink); + qLinkGstElements(m_audioConvert, m_appSink); + }); } void QGstreamerAudioDecoder::removeAppSink() @@ -516,43 +484,48 @@ void QGstreamerAudioDecoder::removeAppSink() if (!m_appSink) return; - gst_element_unlink(m_audioConvert.element(), GST_ELEMENT(m_appSink)); - gst_bin_remove(m_outputBin.bin(), GST_ELEMENT(m_appSink)); + qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::removeAppSink"; - m_appSink = nullptr; + QGstPipeline::modifyPipelineWhileNotRunning(m_playbin.getPipeline(), [&] { + qUnlinkGstElements(m_audioConvert, m_appSink); + m_outputBin.stopAndRemoveElements(m_appSink); + }); + m_appSink = {}; } void QGstreamerAudioDecoder::updateDuration() { - int duration = -1; - - if (!m_playbin.isNull()) - duration = m_playbin.duration() / 1000000; + std::optional<std::chrono::milliseconds> duration = m_playbin.durationInMs(); + if (!duration) + duration = invalidDuration; if (m_duration != duration) { - m_duration = duration; - emit durationChanged(m_duration); + m_duration = *duration; + durationChanged(m_duration.count()); } - if (m_duration > 0) + if (m_duration.count() > 0) m_durationQueries = 0; if (m_durationQueries > 0) { //increase delay between duration requests int delay = 25 << (5 - m_durationQueries); - QTimer::singleShot(delay, this, SLOT(updateDuration())); + QTimer::singleShot(delay, this, &QGstreamerAudioDecoder::updateDuration); m_durationQueries--; } } -qint64 QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer* buffer) +std::chrono::nanoseconds QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer *buffer) { - qint64 position = GST_BUFFER_TIMESTAMP(buffer); - if (position >= 0) - position = position / G_GINT64_CONSTANT(1000); // microseconds + using namespace std::chrono; + using namespace std::chrono_literals; + nanoseconds position{ GST_BUFFER_TIMESTAMP(buffer) }; + if (position >= 0ns) + return position; else - position = -1; - return position; + return invalidPosition; } QT_END_NAMESPACE + +#include "moc_qgstreameraudiodecoder_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h index 85ff53041..d2d259dde 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIODECODERCONTROL_H #define QGSTREAMERAUDIODECODERCONTROL_H @@ -51,34 +15,30 @@ // We mean it. // +#include <QtMultimedia/private/qmultimediautils_p.h> +#include <QtMultimedia/private/qplatformaudiodecoder_p.h> #include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include <QObject> +#include <QtMultimedia/qaudiodecoder.h> +#include <QtCore/qobject.h> #include <QtCore/qmutex.h> #include <QtCore/qurl.h> -#include "private/qplatformaudiodecoder_p.h" -#include <qgstpipeline_p.h> -#include "qaudiodecoder.h" +#include <common/qgst_p.h> +#include <common/qgstappsource_p.h> +#include <common/qgstpipeline_p.h> -#if QT_CONFIG(gstreamer_app) -#include <qgstappsrc_p.h> -#endif - -#include <qgst_p.h> #include <gst/app/gstappsink.h> QT_BEGIN_NAMESPACE class QGstreamerMessage; -class QGstreamerAudioDecoder - : public QPlatformAudioDecoder, - public QGstreamerBusMessageFilter +class QGstreamerAudioDecoder final : public QPlatformAudioDecoder, public QGstreamerBusMessageFilter { Q_OBJECT public: - QGstreamerAudioDecoder(QAudioDecoder *parent); + static QMaybe<QPlatformAudioDecoder *> create(QAudioDecoder *parent); virtual ~QGstreamerAudioDecoder(); QUrl source() const override; @@ -94,7 +54,6 @@ public: void setAudioFormat(const QAudioFormat &format) override; QAudioBuffer read() override; - bool bufferAvailable() const override; qint64 position() const override; qint64 duration() const override; @@ -102,41 +61,49 @@ public: // GStreamerBusMessageFilter interface bool processBusMessage(const QGstreamerMessage &message) override; -#if QT_CONFIG(gstreamer_app) - QGstAppSrc *appsrc() const { return m_appSrc; } - static void configureAppSrcElement(GObject*, GObject*, GParamSpec*, QGstreamerAudioDecoder *_this); -#endif - - static GstFlowReturn new_sample(GstAppSink *sink, gpointer user_data); - private slots: void updateDuration(); private: + explicit QGstreamerAudioDecoder(QAudioDecoder *parent); + + static GstFlowReturn new_sample(GstAppSink *sink, gpointer user_data); + GstFlowReturn newSample(GstAppSink *sink); + + static void configureAppSrcElement(GObject *, GObject *, GParamSpec *, + QGstreamerAudioDecoder *_this); + void setAudioFlags(bool wantNativeAudio); void addAppSink(); void removeAppSink(); - void processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString); - static qint64 getPositionFromBuffer(GstBuffer* buffer); + bool handlePlaybinMessage(const QGstreamerMessage &); + + void processInvalidMedia(QAudioDecoder::Error errorCode, const QString &errorString); + static std::chrono::nanoseconds getPositionFromBuffer(GstBuffer *buffer); QGstPipeline m_playbin; QGstBin m_outputBin; QGstElement m_audioConvert; - GstAppSink *m_appSink = nullptr; - QGstAppSrc *m_appSrc = nullptr; + QGstAppSink m_appSink; + QGstAppSource *m_appSrc = nullptr; QUrl mSource; QIODevice *mDevice = nullptr; QAudioFormat mFormat; - mutable QMutex m_buffersMutex; int m_buffersAvailable = 0; - qint64 m_position = -1; - qint64 m_duration = -1; + static constexpr auto invalidDuration = std::chrono::milliseconds{ -1 }; + static constexpr auto invalidPosition = std::chrono::milliseconds{ -1 }; + std::chrono::milliseconds m_position{ invalidPosition }; + std::chrono::milliseconds m_duration{ invalidDuration }; int m_durationQueries = 0; + + qint32 m_currentSessionId{}; + + QGObjectHandlerScopedConnection m_deepNotifySourceConnection; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp index aaac46efd..b22e40118 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp @@ -1,61 +1,28 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreameraudiodevice_p.h" -#include <qgstutils_p.h> +#include <common/qgst_p.h> +#include <common/qgstutils_p.h> #include <private/qplatformmediaintegration_p.h> QT_BEGIN_NAMESPACE -QGStreamerAudioDeviceInfo::QGStreamerAudioDeviceInfo(GstDevice *d, const QByteArray &device, QAudioDevice::Mode mode) +QGStreamerAudioDeviceInfo::QGStreamerAudioDeviceInfo(GstDevice *d, const QByteArray &device, + QAudioDevice::Mode mode) : QAudioDevicePrivate(device, mode), - gstDevice(d) + gstDevice{ + d, + QGstDeviceHandle::NeedsRef, + } { - Q_ASSERT(gstDevice); - gst_object_ref(gstDevice); + QGString name{ + gst_device_get_display_name(gstDevice.get()), + }; + description = name.toQString(); - auto *n = gst_device_get_display_name(gstDevice); - description = QString::fromUtf8(n); - g_free(n); - - QGstCaps caps = gst_device_get_caps(gstDevice); + auto caps = QGstCaps(gst_device_get_caps(gstDevice.get()), QGstCaps::HasRef); int size = caps.size(); for (int i = 0; i < size; ++i) { auto c = caps.at(i); @@ -82,10 +49,39 @@ QGStreamerAudioDeviceInfo::QGStreamerAudioDeviceInfo(GstDevice *d, const QByteAr preferredFormat.setSampleFormat(f); } -QGStreamerAudioDeviceInfo::~QGStreamerAudioDeviceInfo() +QGStreamerCustomAudioDeviceInfo::QGStreamerCustomAudioDeviceInfo( + const QByteArray &gstreamerPipeline, QAudioDevice::Mode mode) + : QAudioDevicePrivate{ + gstreamerPipeline, + mode, + } +{ +} + +QAudioDevice qMakeCustomGStreamerAudioInput(const QByteArray &gstreamerPipeline) +{ + auto deviceInfo = std::make_unique<QGStreamerCustomAudioDeviceInfo>(gstreamerPipeline, + QAudioDevice::Mode::Input); + + return deviceInfo.release()->create(); +} + +QAudioDevice qMakeCustomGStreamerAudioOutput(const QByteArray &gstreamerPipeline) +{ + auto deviceInfo = std::make_unique<QGStreamerCustomAudioDeviceInfo>(gstreamerPipeline, + QAudioDevice::Mode::Output); + + return deviceInfo.release()->create(); +} + +bool isCustomAudioDevice(const QAudioDevicePrivate *device) +{ + return dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(device); +} + +bool isCustomAudioDevice(const QAudioDevice &device) { - if (gstDevice) - gst_object_unref(gstDevice); + return isCustomAudioDevice(device.handle()); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h index 9da109e38..403fd5e74 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIODEVICEINFO_H #define QGSTREAMERAUDIODEVICEINFO_H @@ -55,9 +19,11 @@ #include <QtCore/qstringlist.h> #include <QtCore/qlist.h> -#include "qaudio.h" -#include "qaudiodevice.h" -#include <private/qaudiodevice_p.h> +#include <QtMultimedia/qaudio.h> +#include <QtMultimedia/qaudiodevice.h> +#include <QtMultimedia/private/qaudiodevice_p.h> + +#include <QtQGstreamerMediaPlugin/private/qgst_handle_types_p.h> #include <gst/gst.h> @@ -67,11 +33,22 @@ class QGStreamerAudioDeviceInfo : public QAudioDevicePrivate { public: QGStreamerAudioDeviceInfo(GstDevice *gstDevice, const QByteArray &device, QAudioDevice::Mode mode); - ~QGStreamerAudioDeviceInfo(); - GstDevice *gstDevice = nullptr; + QGstDeviceHandle gstDevice; }; +class QGStreamerCustomAudioDeviceInfo : public QAudioDevicePrivate +{ +public: + QGStreamerCustomAudioDeviceInfo(const QByteArray &gstreamerPipeline, QAudioDevice::Mode mode); +}; + +bool isCustomAudioDevice(const QAudioDevicePrivate *device); +bool isCustomAudioDevice(const QAudioDevice &device); + +QAudioDevice qMakeCustomGStreamerAudioInput(const QByteArray &gstreamerPipeline); +QAudioDevice qMakeCustomGStreamerAudioOutput(const QByteArray &gstreamerPipeline); + QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink.cpp deleted file mode 100644 index f3602186c..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink.cpp +++ /dev/null @@ -1,397 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QtCore/qcoreapplication.h> -#include <QtCore/qdebug.h> -#include <QtCore/qmath.h> -#include <private/qaudiohelpers_p.h> - -#include "qgstreameraudiosink_p.h" -#include "qgstreameraudiodevice_p.h" -#include <sys/types.h> -#include <unistd.h> - -#include <qgstpipeline_p.h> -#include <qgstappsrc_p.h> - -#include <qgstutils_p.h> -#include <qgstreamermessage_p.h> - -QT_BEGIN_NAMESPACE - -QGStreamerAudioSink::QGStreamerAudioSink(const QAudioDevice &device) - : m_device(device.id()), - gstPipeline("pipeline") -{ - gstPipeline.installMessageFilter(this); - - m_appSrc = new QGstAppSrc; - connect(m_appSrc, &QGstAppSrc::bytesProcessed, this, &QGStreamerAudioSink::bytesProcessedByAppSrc); - connect(m_appSrc, &QGstAppSrc::noMoreData, this, &QGStreamerAudioSink::needData); - gstAppSrc = m_appSrc->element(); - - // gstDecodeBin = gst_element_factory_make ("decodebin", "dec"); - QGstElement queue("queue", "queue"); - QGstElement conv("audioconvert", "conv"); - gstVolume = QGstElement("volume", "volume"); - if (m_volume != 1.) - gstVolume.set("volume", m_volume); - - // link decodeBin to audioconvert in a callback once we get a pad from the decoder - // g_signal_connect (gstDecodeBin, "pad-added", (GCallback) padAdded, conv); - - const auto *audioInfo = static_cast<const QGStreamerAudioDeviceInfo *>(device.handle()); - gstOutput = gst_device_create_element(audioInfo->gstDevice, nullptr); - - gstPipeline.add(gstAppSrc, queue, /*gstDecodeBin, */ conv, gstVolume, gstOutput); - gstAppSrc.link(queue, conv, gstVolume, gstOutput); -} - -QGStreamerAudioSink::~QGStreamerAudioSink() -{ - close(); - gstPipeline = {}; - gstVolume = {}; - gstAppSrc = {}; - delete m_appSrc; - m_appSrc = nullptr; -} - -void QGStreamerAudioSink::setError(QAudio::Error error) -{ - if (m_errorState == error) - return; - - m_errorState = error; - emit errorChanged(error); -} - -QAudio::Error QGStreamerAudioSink::error() const -{ - return m_errorState; -} - -void QGStreamerAudioSink::setState(QAudio::State state) -{ - if (m_deviceState == state) - return; - - m_deviceState = state; - emit stateChanged(state); -} - -QAudio::State QGStreamerAudioSink::state() const -{ - return m_deviceState; -} - -void QGStreamerAudioSink::start(QIODevice *device) -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!m_format.isValid()) { - setError(QAudio::OpenError); - return; - } - - m_pullMode = true; - m_audioSource = device; - - if (!open()) { - m_audioSource = nullptr; - setError(QAudio::OpenError); - return; - } - - setState(QAudio::ActiveState); -} - -QIODevice *QGStreamerAudioSink::start() -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!m_format.isValid()) { - setError(QAudio::OpenError); - return nullptr; - } - - m_pullMode = false; - - if (!open()) - return nullptr; - - m_audioSource = new GStreamerOutputPrivate(this); - m_audioSource->open(QIODevice::WriteOnly|QIODevice::Unbuffered); - - setState(QAudio::IdleState); - - return m_audioSource; -} - -#if 0 -static void padAdded(GstElement *element, GstPad *pad, gpointer data) -{ - GstElement *other = static_cast<GstElement *>(data); - - gchar *name = gst_pad_get_name(pad); - qDebug("A new pad %s was created for %s\n", name, gst_element_get_name(element)); - g_free(name); - - qDebug("element %s will be linked to %s\n", - gst_element_get_name(element), - gst_element_get_name(other)); - gst_element_link(element, other); -} -#endif - -bool QGStreamerAudioSink::processBusMessage(const QGstreamerMessage &message) -{ - auto *msg = message.rawMessage(); - switch (GST_MESSAGE_TYPE (msg)) { - case GST_MESSAGE_EOS: - setState(QAudio::IdleState); - break; - case GST_MESSAGE_ERROR: { - setError(QAudio::IOError); - gchar *debug; - GError *error; - - gst_message_parse_error (msg, &error, &debug); - g_free (debug); - - qDebug("Error: %s\n", error->message); - g_error_free (error); - - break; - } - default: - break; - } - - return true; -} - -bool QGStreamerAudioSink::open() -{ - if (m_opened) - return true; - - if (gstOutput.isNull()) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - -// qDebug() << "GST caps:" << gst_caps_to_string(caps); - m_appSrc->setup(m_audioSource, m_audioSource ? m_audioSource->pos() : 0); - m_appSrc->setAudioFormat(m_format); - - /* run */ - gstPipeline.setState(GST_STATE_PLAYING); - - m_opened = true; - - m_timeStamp.restart(); - m_bytesProcessed = 0; - - return true; -} - -void QGStreamerAudioSink::close() -{ - if (!m_opened) - return; - - if (!gstPipeline.setStateSync(GST_STATE_NULL)) - qWarning() << "failed to close the audio output stream"; - - if (!m_pullMode && m_audioSource) - delete m_audioSource; - m_audioSource = nullptr; - m_opened = false; -} - -qint64 QGStreamerAudioSink::write(const char *data, qint64 len) -{ - if (!len) - return 0; - if (m_errorState == QAudio::UnderrunError) - m_errorState = QAudio::NoError; - - m_appSrc->write(data, len); - return len; -} - -void QGStreamerAudioSink::stop() -{ - if (m_deviceState == QAudio::StoppedState) - return; - - close(); - - setError(QAudio::NoError); - setState(QAudio::StoppedState); -} - -qsizetype QGStreamerAudioSink::bytesFree() const -{ - if (m_deviceState != QAudio::ActiveState && m_deviceState != QAudio::IdleState) - return 0; - - return m_appSrc->canAcceptMoreData() ? 4096*4 : 0; -} - -void QGStreamerAudioSink::setBufferSize(qsizetype value) -{ - m_bufferSize = value; - if (!gstAppSrc.isNull()) - gst_app_src_set_max_bytes(GST_APP_SRC(gstAppSrc.element()), value); -} - -qsizetype QGStreamerAudioSink::bufferSize() const -{ - return m_bufferSize; -} - -qint64 QGStreamerAudioSink::processedUSecs() const -{ - qint64 result = qint64(1000000) * m_bytesProcessed / - m_format.bytesPerFrame() / - m_format.sampleRate(); - - return result; -} - -void QGStreamerAudioSink::resume() -{ - if (m_deviceState == QAudio::SuspendedState) { - m_appSrc->resume(); - gstPipeline.setState(GST_STATE_PLAYING); - - setState(m_pullMode ? QAudio::ActiveState : QAudio::IdleState); - setError(QAudio::NoError); - } -} - -void QGStreamerAudioSink::setFormat(const QAudioFormat &format) -{ - m_format = format; -} - -QAudioFormat QGStreamerAudioSink::format() const -{ - return m_format; -} - -void QGStreamerAudioSink::suspend() -{ - if (m_deviceState == QAudio::ActiveState || m_deviceState == QAudio::IdleState) { - setError(QAudio::NoError); - setState(QAudio::SuspendedState); - - gstPipeline.setState(GST_STATE_PAUSED); - m_appSrc->suspend(); - // ### elapsed time - } -} - -void QGStreamerAudioSink::reset() -{ - stop(); -} - -GStreamerOutputPrivate::GStreamerOutputPrivate(QGStreamerAudioSink *audio) -{ - m_audioDevice = audio; -} - -qint64 GStreamerOutputPrivate::readData(char *data, qint64 len) -{ - Q_UNUSED(data); - Q_UNUSED(len); - - return 0; -} - -qint64 GStreamerOutputPrivate::writeData(const char *data, qint64 len) -{ - if (m_audioDevice->state() == QAudio::IdleState) - m_audioDevice->setState(QAudio::ActiveState); - return m_audioDevice->write(data, len); -} - -void QGStreamerAudioSink::setVolume(qreal vol) -{ - if (m_volume == vol) - return; - - m_volume = vol; - if (!gstVolume.isNull()) - gstVolume.set("volume", vol); -} - -qreal QGStreamerAudioSink::volume() const -{ - return m_volume; -} - -void QGStreamerAudioSink::bytesProcessedByAppSrc(int bytes) -{ - m_bytesProcessed += bytes; - setState(QAudio::ActiveState); - setError(QAudio::NoError); -} - -void QGStreamerAudioSink::needData() -{ - if (state() != QAudio::StoppedState && state() != QAudio::IdleState) { - setState(QAudio::IdleState); - setError(m_audioSource && m_audioSource->atEnd() ? QAudio::NoError : QAudio::UnderrunError); - } -} - -QT_END_NAMESPACE - -#include "moc_qgstreameraudiosink_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink_p.h deleted file mode 100644 index 080fe1cbf..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink_p.h +++ /dev/null @@ -1,157 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QAUDIOOUTPUTGSTREAMER_H -#define QAUDIOOUTPUTGSTREAMER_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <QtCore/qfile.h> -#include <QtCore/qtimer.h> -#include <QtCore/qstring.h> -#include <QtCore/qstringlist.h> -#include <QtCore/qelapsedtimer.h> -#include <QtCore/qiodevice.h> -#include <QtCore/private/qringbuffer_p.h> - -#include "qaudio.h" -#include "qaudiodevice.h" -#include <private/qaudiosystem_p.h> - -#include <qgst_p.h> -#include <qgstpipeline_p.h> - -QT_BEGIN_NAMESPACE - -class QGstAppSrc; - -class QGStreamerAudioSink - : public QPlatformAudioSink, - public QGstreamerBusMessageFilter -{ - friend class GStreamerOutputPrivate; - Q_OBJECT - -public: - QGStreamerAudioSink(const QAudioDevice &device); - ~QGStreamerAudioSink(); - - void start(QIODevice *device) override; - QIODevice *start() override; - void stop() override; - void reset() override; - void suspend() override; - void resume() override; - qsizetype bytesFree() const override; - void setBufferSize(qsizetype value) override; - qsizetype bufferSize() const override; - qint64 processedUSecs() const override; - QAudio::Error error() const override; - QAudio::State state() const override; - void setFormat(const QAudioFormat &format) override; - QAudioFormat format() const override; - - void setVolume(qreal volume) override; - qreal volume() const override; - -private Q_SLOTS: - void bytesProcessedByAppSrc(int bytes); - void needData(); - -private: - void setState(QAudio::State state); - void setError(QAudio::Error error); - - bool processBusMessage(const QGstreamerMessage &message) override; - - bool open(); - void close(); - qint64 write(const char *data, qint64 len); - -private: - QByteArray m_device; - QAudioFormat m_format; - QAudio::Error m_errorState = QAudio::NoError; - QAudio::State m_deviceState = QAudio::StoppedState; - bool m_pullMode = true; - bool m_opened = false; - QIODevice *m_audioSource = nullptr; - QTimer m_periodTimer; - int m_bufferSize = 0; - qint64 m_bytesProcessed = 0; - QElapsedTimer m_timeStamp; - qreal m_volume = 1.; - QByteArray pushData; - - QGstPipeline gstPipeline; - QGstElement gstOutput; - QGstElement gstVolume; - QGstElement gstAppSrc; - QGstAppSrc *m_appSrc = nullptr; -}; - -class GStreamerOutputPrivate : public QIODevice -{ - friend class QGStreamerAudioSink; - Q_OBJECT - -public: - GStreamerOutputPrivate(QGStreamerAudioSink *audio); - virtual ~GStreamerOutputPrivate() {} - -protected: - qint64 readData(char *data, qint64 len) override; - qint64 writeData(const char *data, qint64 len) override; - -private: - QGStreamerAudioSink *m_audioDevice; -}; - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource.cpp deleted file mode 100644 index ffa402627..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QtCore/qcoreapplication.h> -#include <QtCore/qdebug.h> -#include <QtCore/qmath.h> -#include <private/qaudiohelpers_p.h> - -#include "qgstreameraudiosource_p.h" -#include "qgstreameraudiodevice_p.h" -#include <sys/types.h> -#include <unistd.h> - -#include <gst/gst.h> -Q_DECLARE_OPAQUE_POINTER(GstSample *); -Q_DECLARE_METATYPE(GstSample *); - -QT_BEGIN_NAMESPACE - - -QGStreamerAudioSource::QGStreamerAudioSource(const QAudioDevice &device) - : m_info(device), - m_device(device.id()) -{ - qRegisterMetaType<GstSample *>(); -} - -QGStreamerAudioSource::~QGStreamerAudioSource() -{ - close(); -} - -void QGStreamerAudioSource::setError(QAudio::Error error) -{ - if (m_errorState == error) - return; - - m_errorState = error; - emit errorChanged(error); -} - -QAudio::Error QGStreamerAudioSource::error() const -{ - return m_errorState; -} - -void QGStreamerAudioSource::setState(QAudio::State state) -{ - if (m_deviceState == state) - return; - - m_deviceState = state; - emit stateChanged(state); -} - -QAudio::State QGStreamerAudioSource::state() const -{ - return m_deviceState; -} - -void QGStreamerAudioSource::setFormat(const QAudioFormat &format) -{ - if (m_deviceState == QAudio::StoppedState) - m_format = format; -} - -QAudioFormat QGStreamerAudioSource::format() const -{ - return m_format; -} - -void QGStreamerAudioSource::start(QIODevice *device) -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!open()) - return; - - m_pullMode = true; - m_audioSink = device; - - setState(QAudio::ActiveState); -} - -QIODevice *QGStreamerAudioSource::start() -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!open()) - return nullptr; - - m_pullMode = false; - m_audioSink = new GStreamerInputPrivate(this); - m_audioSink->open(QIODevice::ReadOnly | QIODevice::Unbuffered); - - setState(QAudio::IdleState); - - return m_audioSink; -} - -void QGStreamerAudioSource::stop() -{ - if (m_deviceState == QAudio::StoppedState) - return; - - close(); - - setError(QAudio::NoError); - setState(QAudio::StoppedState); -} - -bool QGStreamerAudioSource::open() -{ - if (m_opened) - return true; - - const auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_info.handle()); - if (!deviceInfo->gstDevice) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - - gstInput = QGstElement(gst_device_create_element(deviceInfo->gstDevice, nullptr)); - if (gstInput.isNull()) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - - auto gstCaps = QGstUtils::capsForAudioFormat(m_format); - - if (gstCaps.isNull()) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - - -#ifdef DEBUG_AUDIO - qDebug() << "Opening input" << QTime::currentTime(); - qDebug() << "Caps: " << gst_caps_to_string(gstCaps); -#endif - - gstPipeline = QGstPipeline("pipeline"); - - auto *gstBus = gst_pipeline_get_bus(gstPipeline.pipeline()); - gst_bus_add_watch(gstBus, &QGStreamerAudioSource::busMessage, this); - gst_object_unref (gstBus); - - gstAppSink = createAppSink(); - gstAppSink.set("caps", gstCaps); - - QGstElement conv("audioconvert", "conv"); - gstVolume = QGstElement("volume", "volume"); - if (m_volume != 1.) - gstVolume.set("volume", m_volume); - - gstPipeline.add(gstInput, gstVolume, conv, gstAppSink); - gstInput.link(gstVolume, conv, gstAppSink); - - gstPipeline.setState(GST_STATE_PLAYING); - - m_opened = true; - - m_timeStamp.restart(); - m_elapsedTimeOffset = 0; - m_bytesWritten = 0; - - return true; -} - -void QGStreamerAudioSource::close() -{ - if (!m_opened) - return; - - gstPipeline.setState(GST_STATE_NULL); - gstPipeline = {}; - gstVolume = {}; - gstAppSink = {}; - gstInput = {}; - - if (!m_pullMode && m_audioSink) { - delete m_audioSink; - } - m_audioSink = nullptr; - m_opened = false; -} - -gboolean QGStreamerAudioSource::busMessage(GstBus *, GstMessage *msg, gpointer user_data) -{ - QGStreamerAudioSource *input = static_cast<QGStreamerAudioSource *>(user_data); - switch (GST_MESSAGE_TYPE (msg)) { - case GST_MESSAGE_EOS: - input->stop(); - break; - case GST_MESSAGE_ERROR: { - input->setError(QAudio::IOError); - gchar *debug; - GError *error; - - gst_message_parse_error (msg, &error, &debug); - g_free (debug); - - qDebug("Error: %s\n", error->message); - g_error_free (error); - - break; - } - default: - break; - } - return false; -} - -qsizetype QGStreamerAudioSource::bytesReady() const -{ - return m_buffer.size(); -} - -void QGStreamerAudioSource::resume() -{ - if (m_deviceState == QAudio::SuspendedState || m_deviceState == QAudio::IdleState) { - gstPipeline.setState(GST_STATE_PLAYING); - setState(QAudio::ActiveState); - setError(QAudio::NoError); - } -} - -void QGStreamerAudioSource::setVolume(qreal vol) -{ - if (m_volume == vol) - return; - - m_volume = vol; - if (!gstVolume.isNull()) - gstVolume.set("volume", vol); -} - -qreal QGStreamerAudioSource::volume() const -{ - return m_volume; -} - -void QGStreamerAudioSource::setBufferSize(qsizetype value) -{ - m_bufferSize = value; -} - -qsizetype QGStreamerAudioSource::bufferSize() const -{ - return m_bufferSize; -} - -qint64 QGStreamerAudioSource::processedUSecs() const -{ - return m_format.durationForBytes(m_bytesWritten); -} - -void QGStreamerAudioSource::suspend() -{ - if (m_deviceState == QAudio::ActiveState) { - setError(QAudio::NoError); - setState(QAudio::SuspendedState); - - gstPipeline.setState(GST_STATE_PAUSED); - } -} - -void QGStreamerAudioSource::reset() -{ - stop(); - m_buffer.clear(); -} - -//#define MAX_BUFFERS_IN_QUEUE 4 - -QGstElement QGStreamerAudioSource::createAppSink() -{ - QGstElement sink("appsink", "appsink"); - GstAppSink *appSink = reinterpret_cast<GstAppSink *>(sink.element()); - - GstAppSinkCallbacks callbacks; - memset(&callbacks, 0, sizeof(callbacks)); - callbacks.eos = &eos; - callbacks.new_sample = &new_sample; - gst_app_sink_set_callbacks(appSink, &callbacks, this, nullptr); -// gst_app_sink_set_max_buffers(appSink, MAX_BUFFERS_IN_QUEUE); - gst_base_sink_set_sync(GST_BASE_SINK(appSink), FALSE); - - return sink; -} - -void QGStreamerAudioSource::newDataAvailable(GstSample *sample) -{ - if (m_audioSink) { - GstBuffer *buffer = gst_sample_get_buffer(sample); - GstMapInfo mapInfo; - gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); - const char *bufferData = (const char*)mapInfo.data; - gsize bufferSize = mapInfo.size; - - if (!m_pullMode) { - // need to store that data in the QBuffer - m_buffer.append(bufferData, bufferSize); - m_audioSink->readyRead(); - } else { - m_bytesWritten += bufferSize; - m_audioSink->write(bufferData, bufferSize); - } - - gst_buffer_unmap(buffer, &mapInfo); - } - - gst_sample_unref(sample); -} - -GstFlowReturn QGStreamerAudioSource::new_sample(GstAppSink *sink, gpointer user_data) -{ - // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()." - QGStreamerAudioSource *control = static_cast<QGStreamerAudioSource*>(user_data); - - GstSample *sample = gst_app_sink_pull_sample(sink); - QMetaObject::invokeMethod(control, "newDataAvailable", Qt::AutoConnection, Q_ARG(GstSample *, sample)); - - return GST_FLOW_OK; -} - -void QGStreamerAudioSource::eos(GstAppSink *, gpointer user_data) -{ - QGStreamerAudioSource *control = static_cast<QGStreamerAudioSource*>(user_data); - control->setState(QAudio::StoppedState); -} - -GStreamerInputPrivate::GStreamerInputPrivate(QGStreamerAudioSource *audio) -{ - m_audioDevice = qobject_cast<QGStreamerAudioSource*>(audio); -} - -qint64 GStreamerInputPrivate::readData(char *data, qint64 len) -{ - if (m_audioDevice->state() == QAudio::IdleState) - m_audioDevice->setState(QAudio::ActiveState); - qint64 bytes = m_audioDevice->m_buffer.read(data, len); - m_audioDevice->m_bytesWritten += bytes; - return bytes; -} - -qint64 GStreamerInputPrivate::writeData(const char *data, qint64 len) -{ - Q_UNUSED(data); - Q_UNUSED(len); - return 0; -} - -qint64 GStreamerInputPrivate::bytesAvailable() const -{ - return m_audioDevice->m_buffer.size(); -} - - -QT_END_NAMESPACE - -#include "moc_qgstreameraudiosource_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource_p.h deleted file mode 100644 index 5a04702ca..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource_p.h +++ /dev/null @@ -1,159 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists for the convenience -// of other Qt classes. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#ifndef QAUDIOINPUTGSTREAMER_H -#define QAUDIOINPUTGSTREAMER_H - -#include <QtCore/qfile.h> -#include <QtCore/qtimer.h> -#include <QtCore/qstring.h> -#include <QtCore/qstringlist.h> -#include <QtCore/qelapsedtimer.h> -#include <QtCore/qiodevice.h> -#include <QtCore/qmutex.h> -#include <QtCore/qatomic.h> -#include <QtCore/private/qringbuffer_p.h> - -#include "qaudio.h" -#include "qaudiodevice.h" -#include <private/qaudiosystem_p.h> - -#include <qgstutils_p.h> -#include <qgstpipeline_p.h> -#include <gst/app/gstappsink.h> - -QT_BEGIN_NAMESPACE - -class GStreamerInputPrivate; - -class QGStreamerAudioSource - : public QPlatformAudioSource -{ - Q_OBJECT - friend class GStreamerInputPrivate; -public: - QGStreamerAudioSource(const QAudioDevice &device); - ~QGStreamerAudioSource(); - - void start(QIODevice *device) override; - QIODevice *start() override; - void stop() override; - void reset() override; - void suspend() override; - void resume() override; - qsizetype bytesReady() const override; - void setBufferSize(qsizetype value) override; - qsizetype bufferSize() const override; - qint64 processedUSecs() const override; - QAudio::Error error() const override; - QAudio::State state() const override; - void setFormat(const QAudioFormat &format) override; - QAudioFormat format() const override; - - void setVolume(qreal volume) override; - qreal volume() const override; - -private Q_SLOTS: - void newDataAvailable(GstSample *sample); - -private: - void setState(QAudio::State state); - void setError(QAudio::Error error); - - QGstElement createAppSink(); - static GstFlowReturn new_sample(GstAppSink *, gpointer user_data); - static void eos(GstAppSink *, gpointer user_data); - - bool open(); - void close(); - - static gboolean busMessage(GstBus *bus, GstMessage *msg, gpointer user_data); - - QAudioDevice m_info; - qint64 m_bytesWritten = 0; - QIODevice *m_audioSink = nullptr; - QAudioFormat m_format; - QAudio::Error m_errorState = QAudio::NoError; - QAudio::State m_deviceState = QAudio::StoppedState; - qreal m_volume = 1.; - - QRingBuffer m_buffer; - QAtomicInteger<bool> m_pullMode = true; - bool m_opened = false; - int m_bufferSize = 0; - qint64 m_elapsedTimeOffset = 0; - QElapsedTimer m_timeStamp; - QByteArray m_device; - QByteArray m_tempBuffer; - - QGstElement gstInput; - QGstPipeline gstPipeline; - QGstElement gstVolume; - QGstElement gstAppSink; -}; - -class GStreamerInputPrivate : public QIODevice -{ - Q_OBJECT -public: - GStreamerInputPrivate(QGStreamerAudioSource *audio); - ~GStreamerInputPrivate() {}; - - qint64 readData(char *data, qint64 len) override; - qint64 writeData(const char *data, qint64 len) override; - qint64 bytesAvailable() const override; - bool isSequential() const override { return true; } -private: - QGStreamerAudioSource *m_audioDevice; -}; - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/gstreamer/common/qglist_helper_p.h b/src/plugins/multimedia/gstreamer/common/qglist_helper_p.h new file mode 100644 index 000000000..54108e1c3 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qglist_helper_p.h @@ -0,0 +1,82 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGLIST_HELPER_P_H +#define QGLIST_HELPER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/qtconfigmacros.h> + +#include <glib.h> +#include <iterator> + +QT_BEGIN_NAMESPACE + +namespace QGstUtils { + +template <typename ListType> +struct GListIterator +{ + explicit GListIterator(const GList *element = nullptr) : element(element) { } + + const ListType &operator*() const noexcept { return *operator->(); } + const ListType *operator->() const noexcept + { + return reinterpret_cast<const ListType *>(&element->data); + } + + GListIterator &operator++() noexcept + { + if (element) + element = element->next; + + return *this; + } + GListIterator operator++(int n) noexcept + { + for (int i = 0; i != n; ++i) + operator++(); + + return *this; + } + + bool operator==(const GListIterator &r) const noexcept { return element == r.element; } + bool operator!=(const GListIterator &r) const noexcept { return element != r.element; } + + using difference_type = std::ptrdiff_t; + using value_type = ListType; + using pointer = value_type *; + using reference = value_type &; + using iterator_category = std::input_iterator_tag; + + const GList *element = nullptr; +}; + +template <typename ListType> +struct GListRangeAdaptor +{ + static_assert(std::is_pointer_v<ListType>); + + explicit GListRangeAdaptor(const GList *list) : head(list) { } + + auto begin() { return GListIterator<ListType>(head); } + auto end() { return GListIterator<ListType>(nullptr); } + + const GList *head; +}; + +} // namespace QGstUtils + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst.cpp b/src/plugins/multimedia/gstreamer/common/qgst.cpp new file mode 100644 index 000000000..cb1f38495 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst.cpp @@ -0,0 +1,1404 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgst_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamermessage_p.h> + +#include <QtCore/qdebug.h> +#include <QtMultimedia/qcameradevice.h> + +#include <array> + +QT_BEGIN_NAMESPACE + +namespace { + +struct VideoFormat +{ + QVideoFrameFormat::PixelFormat pixelFormat; + GstVideoFormat gstFormat; +}; + +constexpr std::array<VideoFormat, 19> qt_videoFormatLookup{ { + { QVideoFrameFormat::Format_YUV420P, GST_VIDEO_FORMAT_I420 }, + { QVideoFrameFormat::Format_YUV422P, GST_VIDEO_FORMAT_Y42B }, + { QVideoFrameFormat::Format_YV12, GST_VIDEO_FORMAT_YV12 }, + { QVideoFrameFormat::Format_UYVY, GST_VIDEO_FORMAT_UYVY }, + { QVideoFrameFormat::Format_YUYV, GST_VIDEO_FORMAT_YUY2 }, + { QVideoFrameFormat::Format_NV12, GST_VIDEO_FORMAT_NV12 }, + { QVideoFrameFormat::Format_NV21, GST_VIDEO_FORMAT_NV21 }, + { QVideoFrameFormat::Format_AYUV, GST_VIDEO_FORMAT_AYUV }, + { QVideoFrameFormat::Format_Y8, GST_VIDEO_FORMAT_GRAY8 }, + { QVideoFrameFormat::Format_XRGB8888, GST_VIDEO_FORMAT_xRGB }, + { QVideoFrameFormat::Format_XBGR8888, GST_VIDEO_FORMAT_xBGR }, + { QVideoFrameFormat::Format_RGBX8888, GST_VIDEO_FORMAT_RGBx }, + { QVideoFrameFormat::Format_BGRX8888, GST_VIDEO_FORMAT_BGRx }, + { QVideoFrameFormat::Format_ARGB8888, GST_VIDEO_FORMAT_ARGB }, + { QVideoFrameFormat::Format_ABGR8888, GST_VIDEO_FORMAT_ABGR }, + { QVideoFrameFormat::Format_RGBA8888, GST_VIDEO_FORMAT_RGBA }, + { QVideoFrameFormat::Format_BGRA8888, GST_VIDEO_FORMAT_BGRA }, +#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN + { QVideoFrameFormat::Format_Y16, GST_VIDEO_FORMAT_GRAY16_LE }, + { QVideoFrameFormat::Format_P010, GST_VIDEO_FORMAT_P010_10LE }, +#else + { QVideoFrameFormat::Format_Y16, GST_VIDEO_FORMAT_GRAY16_BE }, + { QVideoFrameFormat::Format_P010, GST_VIDEO_FORMAT_P010_10BE }, +#endif +} }; + +int indexOfVideoFormat(QVideoFrameFormat::PixelFormat format) +{ + for (size_t i = 0; i < qt_videoFormatLookup.size(); ++i) + if (qt_videoFormatLookup[i].pixelFormat == format) + return int(i); + + return -1; +} + +int indexOfVideoFormat(GstVideoFormat format) +{ + for (size_t i = 0; i < qt_videoFormatLookup.size(); ++i) + if (qt_videoFormatLookup[i].gstFormat == format) + return int(i); + + return -1; +} + +} // namespace + +// QGValue + +QGValue::QGValue(const GValue *v) : value(v) { } + +bool QGValue::isNull() const +{ + return !value; +} + +std::optional<bool> QGValue::toBool() const +{ + if (!G_VALUE_HOLDS_BOOLEAN(value)) + return std::nullopt; + return g_value_get_boolean(value); +} + +std::optional<int> QGValue::toInt() const +{ + if (!G_VALUE_HOLDS_INT(value)) + return std::nullopt; + return g_value_get_int(value); +} + +std::optional<int> QGValue::toInt64() const +{ + if (!G_VALUE_HOLDS_INT64(value)) + return std::nullopt; + return g_value_get_int64(value); +} + +const char *QGValue::toString() const +{ + return value ? g_value_get_string(value) : nullptr; +} + +std::optional<float> QGValue::getFraction() const +{ + if (!GST_VALUE_HOLDS_FRACTION(value)) + return std::nullopt; + return (float)gst_value_get_fraction_numerator(value) + / (float)gst_value_get_fraction_denominator(value); +} + +std::optional<QGRange<float>> QGValue::getFractionRange() const +{ + if (!GST_VALUE_HOLDS_FRACTION_RANGE(value)) + return std::nullopt; + QGValue min = QGValue{ gst_value_get_fraction_range_min(value) }; + QGValue max = QGValue{ gst_value_get_fraction_range_max(value) }; + return QGRange<float>{ *min.getFraction(), *max.getFraction() }; +} + +std::optional<QGRange<int>> QGValue::toIntRange() const +{ + if (!GST_VALUE_HOLDS_INT_RANGE(value)) + return std::nullopt; + return QGRange<int>{ gst_value_get_int_range_min(value), gst_value_get_int_range_max(value) }; +} + +QGstStructureView QGValue::toStructure() const +{ + if (!value || !GST_VALUE_HOLDS_STRUCTURE(value)) + return QGstStructureView(nullptr); + return QGstStructureView(gst_value_get_structure(value)); +} + +QGstCaps QGValue::toCaps() const +{ + if (!value || !GST_VALUE_HOLDS_CAPS(value)) + return {}; + return QGstCaps(gst_caps_copy(gst_value_get_caps(value)), QGstCaps::HasRef); +} + +bool QGValue::isList() const +{ + return value && GST_VALUE_HOLDS_LIST(value); +} + +int QGValue::listSize() const +{ + return gst_value_list_get_size(value); +} + +QGValue QGValue::at(int index) const +{ + return QGValue{ gst_value_list_get_value(value, index) }; +} + +// QGstStructureView + +QGstStructureView::QGstStructureView(const GstStructure *s) : structure(s) { } + +QGstStructureView::QGstStructureView(const QUniqueGstStructureHandle &handle) + : QGstStructureView{ handle.get() } +{ +} + +QUniqueGstStructureHandle QGstStructureView::clone() const +{ + return QUniqueGstStructureHandle{ gst_structure_copy(structure) }; +} + +bool QGstStructureView::isNull() const +{ + return !structure; +} + +QByteArrayView QGstStructureView::name() const +{ + return gst_structure_get_name(structure); +} + +QGValue QGstStructureView::operator[](const char *fieldname) const +{ + return QGValue{ gst_structure_get_value(structure, fieldname) }; +} + +QGstCaps QGstStructureView::caps() const +{ + return operator[]("caps").toCaps(); +} + +QGstTagListHandle QGstStructureView::tags() const +{ + QGValue tags = operator[]("tags"); + if (tags.isNull()) + return {}; + + QGstTagListHandle tagList; + gst_structure_get(structure, "tags", GST_TYPE_TAG_LIST, &tagList, nullptr); + return tagList; +} + +QSize QGstStructureView::resolution() const +{ + QSize size; + + int w, h; + if (structure && gst_structure_get_int(structure, "width", &w) + && gst_structure_get_int(structure, "height", &h)) { + size.rwidth() = w; + size.rheight() = h; + } + + return size; +} + +QVideoFrameFormat::PixelFormat QGstStructureView::pixelFormat() const +{ + QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid; + + if (!structure) + return pixelFormat; + + if (gst_structure_has_name(structure, "video/x-raw")) { + const gchar *s = gst_structure_get_string(structure, "format"); + if (s) { + GstVideoFormat format = gst_video_format_from_string(s); + int index = indexOfVideoFormat(format); + + if (index != -1) + pixelFormat = qt_videoFormatLookup[index].pixelFormat; + } + } else if (gst_structure_has_name(structure, "image/jpeg")) { + pixelFormat = QVideoFrameFormat::Format_Jpeg; + } + + return pixelFormat; +} + +QGRange<float> QGstStructureView::frameRateRange() const +{ + float minRate = 0.; + float maxRate = 0.; + + if (!structure) + return { 0.f, 0.f }; + + auto extractFraction = [](const GValue *v) -> float { + return (float)gst_value_get_fraction_numerator(v) + / (float)gst_value_get_fraction_denominator(v); + }; + auto extractFrameRate = [&](const GValue *v) { + auto insert = [&](float min, float max) { + if (max > maxRate) + maxRate = max; + if (min < minRate) + minRate = min; + }; + + if (GST_VALUE_HOLDS_FRACTION(v)) { + float rate = extractFraction(v); + insert(rate, rate); + } else if (GST_VALUE_HOLDS_FRACTION_RANGE(v)) { + auto *min = gst_value_get_fraction_range_max(v); + auto *max = gst_value_get_fraction_range_max(v); + insert(extractFraction(min), extractFraction(max)); + } + }; + + const GValue *gstFrameRates = gst_structure_get_value(structure, "framerate"); + if (gstFrameRates) { + if (GST_VALUE_HOLDS_LIST(gstFrameRates)) { + guint nFrameRates = gst_value_list_get_size(gstFrameRates); + for (guint f = 0; f < nFrameRates; ++f) { + extractFrameRate(gst_value_list_get_value(gstFrameRates, f)); + } + } else { + extractFrameRate(gstFrameRates); + } + } else { + const GValue *min = gst_structure_get_value(structure, "min-framerate"); + const GValue *max = gst_structure_get_value(structure, "max-framerate"); + if (min && max) { + minRate = extractFraction(min); + maxRate = extractFraction(max); + } + } + + return { minRate, maxRate }; +} + +QGstreamerMessage QGstStructureView::getMessage() +{ + GstMessage *message = nullptr; + gst_structure_get(structure, "message", GST_TYPE_MESSAGE, &message, nullptr); + return QGstreamerMessage(message, QGstreamerMessage::HasRef); +} + +std::optional<Fraction> QGstStructureView::pixelAspectRatio() const +{ + gint numerator; + gint denominator; + if (gst_structure_get_fraction(structure, "pixel-aspect-ratio", &numerator, &denominator)) { + return Fraction{ + numerator, + denominator, + }; + } + + return std::nullopt; +} + +// QTBUG-125249: gstreamer tries "to keep the input height (because of interlacing)". Can we align +// the behavior between gstreamer and ffmpeg? +static QSize qCalculateFrameSizeGStreamer(QSize resolution, Fraction par) +{ + if (par.numerator == par.denominator || par.numerator < 1 || par.denominator < 1) + return resolution; + + return QSize{ + resolution.width() * par.numerator / par.denominator, + resolution.height(), + }; +} + +QSize QGstStructureView::nativeSize() const +{ + QSize size = resolution(); + if (!size.isValid()) { + qWarning() << Q_FUNC_INFO << "invalid resolution when querying nativeSize"; + return size; + } + + std::optional<Fraction> par = pixelAspectRatio(); + if (par) + size = qCalculateFrameSizeGStreamer(size, *par); + return size; +} + +// QGstCaps + +std::optional<std::pair<QVideoFrameFormat, GstVideoInfo>> QGstCaps::formatAndVideoInfo() const +{ + GstVideoInfo vidInfo; + + bool success = gst_video_info_from_caps(&vidInfo, get()); + if (!success) + return std::nullopt; + + int index = indexOfVideoFormat(vidInfo.finfo->format); + if (index == -1) + return std::nullopt; + + QVideoFrameFormat format(QSize(vidInfo.width, vidInfo.height), + qt_videoFormatLookup[index].pixelFormat); + + if (vidInfo.fps_d > 0) + format.setStreamFrameRate(qreal(vidInfo.fps_n) / vidInfo.fps_d); + + QVideoFrameFormat::ColorRange range = QVideoFrameFormat::ColorRange_Unknown; + switch (vidInfo.colorimetry.range) { + case GST_VIDEO_COLOR_RANGE_UNKNOWN: + break; + case GST_VIDEO_COLOR_RANGE_0_255: + range = QVideoFrameFormat::ColorRange_Full; + break; + case GST_VIDEO_COLOR_RANGE_16_235: + range = QVideoFrameFormat::ColorRange_Video; + break; + } + format.setColorRange(range); + + QVideoFrameFormat::ColorSpace colorSpace = QVideoFrameFormat::ColorSpace_Undefined; + switch (vidInfo.colorimetry.matrix) { + case GST_VIDEO_COLOR_MATRIX_UNKNOWN: + case GST_VIDEO_COLOR_MATRIX_RGB: + case GST_VIDEO_COLOR_MATRIX_FCC: + break; + case GST_VIDEO_COLOR_MATRIX_BT709: + colorSpace = QVideoFrameFormat::ColorSpace_BT709; + break; + case GST_VIDEO_COLOR_MATRIX_BT601: + colorSpace = QVideoFrameFormat::ColorSpace_BT601; + break; + case GST_VIDEO_COLOR_MATRIX_SMPTE240M: + colorSpace = QVideoFrameFormat::ColorSpace_AdobeRgb; + break; + case GST_VIDEO_COLOR_MATRIX_BT2020: + colorSpace = QVideoFrameFormat::ColorSpace_BT2020; + break; + } + format.setColorSpace(colorSpace); + + QVideoFrameFormat::ColorTransfer transfer = QVideoFrameFormat::ColorTransfer_Unknown; + switch (vidInfo.colorimetry.transfer) { + case GST_VIDEO_TRANSFER_UNKNOWN: + break; + case GST_VIDEO_TRANSFER_GAMMA10: + transfer = QVideoFrameFormat::ColorTransfer_Linear; + break; + case GST_VIDEO_TRANSFER_GAMMA22: + case GST_VIDEO_TRANSFER_SMPTE240M: + case GST_VIDEO_TRANSFER_SRGB: + case GST_VIDEO_TRANSFER_ADOBERGB: + transfer = QVideoFrameFormat::ColorTransfer_Gamma22; + break; + case GST_VIDEO_TRANSFER_GAMMA18: + case GST_VIDEO_TRANSFER_GAMMA20: + // not quite, but best fit + case GST_VIDEO_TRANSFER_BT709: + case GST_VIDEO_TRANSFER_BT2020_12: + transfer = QVideoFrameFormat::ColorTransfer_BT709; + break; + case GST_VIDEO_TRANSFER_GAMMA28: + transfer = QVideoFrameFormat::ColorTransfer_Gamma28; + break; + case GST_VIDEO_TRANSFER_LOG100: + case GST_VIDEO_TRANSFER_LOG316: + break; +#if GST_CHECK_VERSION(1, 18, 0) + case GST_VIDEO_TRANSFER_SMPTE2084: + transfer = QVideoFrameFormat::ColorTransfer_ST2084; + break; + case GST_VIDEO_TRANSFER_ARIB_STD_B67: + transfer = QVideoFrameFormat::ColorTransfer_STD_B67; + break; + case GST_VIDEO_TRANSFER_BT2020_10: + transfer = QVideoFrameFormat::ColorTransfer_BT709; + break; + case GST_VIDEO_TRANSFER_BT601: + transfer = QVideoFrameFormat::ColorTransfer_BT601; + break; +#endif + } + format.setColorTransfer(transfer); + + return std::pair{ + std::move(format), + vidInfo, + }; +} + +void QGstCaps::addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, + const char *modifier) +{ + if (!gst_caps_is_writable(get())) + *this = QGstCaps(gst_caps_make_writable(release()), QGstCaps::RefMode::HasRef); + + GValue list = {}; + g_value_init(&list, GST_TYPE_LIST); + + for (QVideoFrameFormat::PixelFormat format : formats) { + int index = indexOfVideoFormat(format); + if (index == -1) + continue; + GValue item = {}; + + g_value_init(&item, G_TYPE_STRING); + g_value_set_string(&item, + gst_video_format_to_string(qt_videoFormatLookup[index].gstFormat)); + gst_value_list_append_value(&list, &item); + g_value_unset(&item); + } + + auto *structure = gst_structure_new("video/x-raw", "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, + INT_MAX, 1, "width", GST_TYPE_INT_RANGE, 1, INT_MAX, + "height", GST_TYPE_INT_RANGE, 1, INT_MAX, nullptr); + gst_structure_set_value(structure, "format", &list); + gst_caps_append_structure(get(), structure); + g_value_unset(&list); + + if (modifier) + gst_caps_set_features(get(), size() - 1, gst_caps_features_from_string(modifier)); +} + +void QGstCaps::setResolution(QSize resolution) +{ + Q_ASSERT(resolution.isValid()); + GValue width{}; + g_value_init(&width, G_TYPE_INT); + g_value_set_int(&width, resolution.width()); + GValue height{}; + g_value_init(&height, G_TYPE_INT); + g_value_set_int(&height, resolution.height()); + + gst_caps_set_value(caps(), "width", &width); + gst_caps_set_value(caps(), "height", &height); +} + +QGstCaps QGstCaps::fromCameraFormat(const QCameraFormat &format) +{ + QSize size = format.resolution(); + GstStructure *structure = nullptr; + if (format.pixelFormat() == QVideoFrameFormat::Format_Jpeg) { + structure = gst_structure_new("image/jpeg", "width", G_TYPE_INT, size.width(), "height", + G_TYPE_INT, size.height(), nullptr); + } else { + int index = indexOfVideoFormat(format.pixelFormat()); + if (index < 0) + return {}; + auto gstFormat = qt_videoFormatLookup[index].gstFormat; + structure = gst_structure_new("video/x-raw", "format", G_TYPE_STRING, + gst_video_format_to_string(gstFormat), "width", G_TYPE_INT, + size.width(), "height", G_TYPE_INT, size.height(), nullptr); + } + auto caps = QGstCaps::create(); + gst_caps_append_structure(caps.get(), structure); + return caps; +} + +QGstCaps QGstCaps::copy() const +{ + return QGstCaps{ + gst_caps_copy(caps()), + QGstCaps::HasRef, + }; +} + +QGstCaps::MemoryFormat QGstCaps::memoryFormat() const +{ + auto *features = gst_caps_get_features(get(), 0); + if (gst_caps_features_contains(features, "memory:GLMemory")) + return GLTexture; + if (gst_caps_features_contains(features, "memory:DMABuf")) + return DMABuf; + return CpuMemory; +} + +int QGstCaps::size() const +{ + return int(gst_caps_get_size(get())); +} + +QGstStructureView QGstCaps::at(int index) const +{ + return QGstStructureView{ + gst_caps_get_structure(get(), index), + }; +} + +GstCaps *QGstCaps::caps() const +{ + return get(); +} + +QGstCaps QGstCaps::create() +{ + return QGstCaps(gst_caps_new_empty(), HasRef); +} + +// QGstObject + +void QGstObject::set(const char *property, const char *str) +{ + g_object_set(get(), property, str, nullptr); +} + +void QGstObject::set(const char *property, bool b) +{ + g_object_set(get(), property, gboolean(b), nullptr); +} + +void QGstObject::set(const char *property, uint i) +{ + g_object_set(get(), property, guint(i), nullptr); +} + +void QGstObject::set(const char *property, int i) +{ + g_object_set(get(), property, gint(i), nullptr); +} + +void QGstObject::set(const char *property, qint64 i) +{ + g_object_set(get(), property, gint64(i), nullptr); +} + +void QGstObject::set(const char *property, quint64 i) +{ + g_object_set(get(), property, guint64(i), nullptr); +} + +void QGstObject::set(const char *property, double d) +{ + g_object_set(get(), property, gdouble(d), nullptr); +} + +void QGstObject::set(const char *property, const QGstObject &o) +{ + g_object_set(get(), property, o.object(), nullptr); +} + +void QGstObject::set(const char *property, const QGstCaps &c) +{ + g_object_set(get(), property, c.caps(), nullptr); +} + +QGString QGstObject::getString(const char *property) const +{ + char *s = nullptr; + g_object_get(get(), property, &s, nullptr); + return QGString(s); +} + +QGstStructureView QGstObject::getStructure(const char *property) const +{ + GstStructure *s = nullptr; + g_object_get(get(), property, &s, nullptr); + return QGstStructureView(s); +} + +bool QGstObject::getBool(const char *property) const +{ + gboolean b = false; + g_object_get(get(), property, &b, nullptr); + return b; +} + +uint QGstObject::getUInt(const char *property) const +{ + guint i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +int QGstObject::getInt(const char *property) const +{ + gint i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +quint64 QGstObject::getUInt64(const char *property) const +{ + guint64 i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +qint64 QGstObject::getInt64(const char *property) const +{ + gint64 i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +float QGstObject::getFloat(const char *property) const +{ + gfloat d = 0; + g_object_get(get(), property, &d, nullptr); + return d; +} + +double QGstObject::getDouble(const char *property) const +{ + gdouble d = 0; + g_object_get(get(), property, &d, nullptr); + return d; +} + +QGstObject QGstObject::getObject(const char *property) const +{ + GstObject *o = nullptr; + g_object_get(get(), property, &o, nullptr); + return QGstObject(o, HasRef); +} + +QGObjectHandlerConnection QGstObject::connect(const char *name, GCallback callback, + gpointer userData) +{ + return QGObjectHandlerConnection{ + *this, + g_signal_connect(get(), name, callback, userData), + }; +} + +void QGstObject::disconnect(gulong handlerId) +{ + g_signal_handler_disconnect(get(), handlerId); +} + +GType QGstObject::type() const +{ + return G_OBJECT_TYPE(get()); +} + +QLatin1StringView QGstObject::typeName() const +{ + return QLatin1StringView{ + g_type_name(type()), + }; +} + +GstObject *QGstObject::object() const +{ + return get(); +} + +QLatin1StringView QGstObject::name() const +{ + using namespace Qt::StringLiterals; + + return get() ? QLatin1StringView{ GST_OBJECT_NAME(get()) } : "(null)"_L1; +} + +// QGObjectHandlerConnection + +QGObjectHandlerConnection::QGObjectHandlerConnection(QGstObject object, gulong handlerId) + : object{ std::move(object) }, handlerId{ handlerId } +{ +} + +void QGObjectHandlerConnection::disconnect() +{ + if (!object) + return; + + object.disconnect(handlerId); + object = {}; + handlerId = invalidHandlerId; +} + +// QGObjectHandlerScopedConnection + +QGObjectHandlerScopedConnection::QGObjectHandlerScopedConnection( + QGObjectHandlerConnection connection) + : connection{ + std::move(connection), + } +{ +} + +QGObjectHandlerScopedConnection::~QGObjectHandlerScopedConnection() +{ + connection.disconnect(); +} + +void QGObjectHandlerScopedConnection::disconnect() +{ + connection.disconnect(); +} + +// QGstPad + +QGstPad::QGstPad(const QGstObject &o) + : QGstPad{ + qGstSafeCast<GstPad>(o.object()), + QGstElement::NeedsRef, + } +{ +} + +QGstPad::QGstPad(GstPad *pad, RefMode mode) + : QGstObject{ + qGstCheckedCast<GstObject>(pad), + mode, + } +{ +} + +QGstCaps QGstPad::currentCaps() const +{ + return QGstCaps(gst_pad_get_current_caps(pad()), QGstCaps::HasRef); +} + +QGstCaps QGstPad::queryCaps() const +{ + return QGstCaps(gst_pad_query_caps(pad(), nullptr), QGstCaps::HasRef); +} + +QGstTagListHandle QGstPad::tags() const +{ + QGstTagListHandle tagList; + g_object_get(object(), "tags", &tagList, nullptr); + return tagList; +} + +std::optional<QPlatformMediaPlayer::TrackType> QGstPad::inferTrackTypeFromName() const +{ + using namespace Qt::Literals; + QLatin1StringView padName = name(); + + if (padName.startsWith("video_"_L1)) + return QPlatformMediaPlayer::TrackType::VideoStream; + if (padName.startsWith("audio_"_L1)) + return QPlatformMediaPlayer::TrackType::AudioStream; + if (padName.startsWith("text_"_L1)) + return QPlatformMediaPlayer::TrackType::SubtitleStream; + + return std::nullopt; +} + +bool QGstPad::isLinked() const +{ + return gst_pad_is_linked(pad()); +} + +bool QGstPad::link(const QGstPad &sink) const +{ + return gst_pad_link(pad(), sink.pad()) == GST_PAD_LINK_OK; +} + +bool QGstPad::unlink(const QGstPad &sink) const +{ + return gst_pad_unlink(pad(), sink.pad()); +} + +bool QGstPad::unlinkPeer() const +{ + return unlink(peer()); +} + +QGstPad QGstPad::peer() const +{ + return QGstPad(gst_pad_get_peer(pad()), HasRef); +} + +QGstElement QGstPad::parent() const +{ + return QGstElement(gst_pad_get_parent_element(pad()), HasRef); +} + +GstPad *QGstPad::pad() const +{ + return qGstCheckedCast<GstPad>(object()); +} + +GstEvent *QGstPad::stickyEvent(GstEventType type) +{ + return gst_pad_get_sticky_event(pad(), type, 0); +} + +bool QGstPad::sendEvent(GstEvent *event) +{ + return gst_pad_send_event(pad(), event); +} + +// QGstClock + +QGstClock::QGstClock(const QGstObject &o) + : QGstClock{ + qGstSafeCast<GstClock>(o.object()), + QGstElement::NeedsRef, + } +{ +} + +QGstClock::QGstClock(GstClock *clock, RefMode mode) + : QGstObject{ + qGstCheckedCast<GstObject>(clock), + mode, + } +{ +} + +GstClock *QGstClock::clock() const +{ + return qGstCheckedCast<GstClock>(object()); +} + +GstClockTime QGstClock::time() const +{ + return gst_clock_get_time(clock()); +} + +// QGstElement + +QGstElement::QGstElement(GstElement *element, RefMode mode) + : QGstObject{ + qGstCheckedCast<GstObject>(element), + mode, + } +{ +} + +QGstElement QGstElement::createFromFactory(const char *factory, const char *name) +{ + GstElement *element = gst_element_factory_make(factory, name); + +#ifndef QT_NO_DEBUG + if (!element) { + qWarning() << "Failed to make element" << name << "from factory" << factory; + return QGstElement{}; + } +#endif + + return QGstElement{ + element, + NeedsRef, + }; +} + +QGstElement QGstElement::createFromFactory(GstElementFactory *factory, const char *name) +{ + return QGstElement{ + gst_element_factory_create(factory, name), + NeedsRef, + }; +} + +QGstElement QGstElement::createFromFactory(const QGstElementFactoryHandle &factory, + const char *name) +{ + return createFromFactory(factory.get(), name); +} + +QGstElement QGstElement::createFromDevice(const QGstDeviceHandle &device, const char *name) +{ + return createFromDevice(device.get(), name); +} + +QGstElement QGstElement::createFromDevice(GstDevice *device, const char *name) +{ + return QGstElement{ + gst_device_create_element(device, name), + QGstElement::NeedsRef, + }; +} + +QGstElement QGstElement::createFromPipelineDescription(const char *str) +{ + QUniqueGErrorHandle error; + QGstElement element{ + gst_parse_launch(str, &error), + QGstElement::NeedsRef, + }; + + if (error) // error does not mean that the element could not be constructed + qWarning() << "gst_parse_launch error:" << error; + + return element; +} + +QGstElement QGstElement::createFromPipelineDescription(const QByteArray &str) +{ + return createFromPipelineDescription(str.constData()); +} + +QGstElementFactoryHandle QGstElement::findFactory(const char *name) +{ + return QGstElementFactoryHandle{ + gst_element_factory_find(name), + QGstElementFactoryHandle::HasRef, + }; +} + +QGstElementFactoryHandle QGstElement::findFactory(const QByteArray &name) +{ + return findFactory(name.constData()); +} + +QGstPad QGstElement::staticPad(const char *name) const +{ + return QGstPad(gst_element_get_static_pad(element(), name), HasRef); +} + +QGstPad QGstElement::src() const +{ + return staticPad("src"); +} + +QGstPad QGstElement::sink() const +{ + return staticPad("sink"); +} + +QGstPad QGstElement::getRequestPad(const char *name) const +{ +#if GST_CHECK_VERSION(1, 19, 1) + return QGstPad(gst_element_request_pad_simple(element(), name), HasRef); +#else + return QGstPad(gst_element_get_request_pad(element(), name), HasRef); +#endif +} + +void QGstElement::releaseRequestPad(const QGstPad &pad) const +{ + return gst_element_release_request_pad(element(), pad.pad()); +} + +GstState QGstElement::state(std::chrono::nanoseconds timeout) const +{ + using namespace std::chrono_literals; + + GstState state; + GstStateChangeReturn change = + gst_element_get_state(element(), &state, nullptr, timeout.count()); + + if (Q_UNLIKELY(change == GST_STATE_CHANGE_ASYNC)) + qWarning() << "QGstElement::state detected an asynchronous state change. Return value not " + "reliable"; + + return state; +} + +GstStateChangeReturn QGstElement::setState(GstState state) +{ + return gst_element_set_state(element(), state); +} + +bool QGstElement::setStateSync(GstState state, std::chrono::nanoseconds timeout) +{ + if (state == GST_STATE_NULL) { + // QTBUG-125251: when changing pipeline state too quickly between NULL->PAUSED->NULL there + // may be a pending task to activate pads while we try to switch to NULL. This can cause an + // assertion failure in gstreamer. we therefore finish the state change when called on a bin + // or pipeline. + if (qIsGstObjectOfType<GstBin>(element())) + finishStateChange(); + } + + GstStateChangeReturn change = gst_element_set_state(element(), state); + if (change == GST_STATE_CHANGE_ASYNC) + change = gst_element_get_state(element(), nullptr, &state, timeout.count()); + + if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) { + qWarning() << "Could not change state of" << name() << "to" << state << change; + dumpPipelineGraph("setStateSyncFailure"); + } + return change == GST_STATE_CHANGE_SUCCESS; +} + +bool QGstElement::syncStateWithParent() +{ + Q_ASSERT(element()); + return gst_element_sync_state_with_parent(element()) == TRUE; +} + +bool QGstElement::finishStateChange(std::chrono::nanoseconds timeout) +{ + GstState state, pending; + GstStateChangeReturn change = + gst_element_get_state(element(), &state, &pending, timeout.count()); + + if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) { + qWarning() << "Could not finish change state of" << name() << change << state << pending; + dumpPipelineGraph("finishStateChangeFailure"); + } + return change == GST_STATE_CHANGE_SUCCESS; +} + +void QGstElement::lockState(bool locked) +{ + gst_element_set_locked_state(element(), locked); +} + +bool QGstElement::isStateLocked() const +{ + return gst_element_is_locked_state(element()); +} + +void QGstElement::sendEvent(GstEvent *event) const +{ + gst_element_send_event(element(), event); +} + +void QGstElement::sendEos() const +{ + sendEvent(gst_event_new_eos()); +} + +std::optional<std::chrono::nanoseconds> QGstElement::duration() const +{ + gint64 d; + if (!gst_element_query_duration(element(), GST_FORMAT_TIME, &d)) { + qDebug() << "QGstElement: failed to query duration"; + return std::nullopt; + } + return std::chrono::nanoseconds{ d }; +} + +std::optional<std::chrono::milliseconds> QGstElement::durationInMs() const +{ + using namespace std::chrono; + auto dur = duration(); + if (dur) + return round<milliseconds>(*dur); + return std::nullopt; +} + +std::optional<std::chrono::nanoseconds> QGstElement::position() const +{ + QGstQueryHandle &query = positionQuery(); + + gint64 pos; + if (gst_element_query(element(), query.get())) { + gst_query_parse_position(query.get(), nullptr, &pos); + return std::chrono::nanoseconds{ pos }; + } + + qDebug() << "QGstElement: failed to query position"; + return std::nullopt; +} + +std::optional<std::chrono::milliseconds> QGstElement::positionInMs() const +{ + using namespace std::chrono; + auto pos = position(); + if (pos) + return round<milliseconds>(*pos); + return std::nullopt; +} + +std::optional<bool> QGstElement::canSeek() const +{ + QGstQueryHandle query{ + gst_query_new_seeking(GST_FORMAT_TIME), + QGstQueryHandle::HasRef, + }; + gboolean canSeek = false; + gst_query_parse_seeking(query.get(), nullptr, &canSeek, nullptr, nullptr); + + if (gst_element_query(element(), query.get())) { + gst_query_parse_seeking(query.get(), nullptr, &canSeek, nullptr, nullptr); + return canSeek; + } + return std::nullopt; +} + +GstClockTime QGstElement::baseTime() const +{ + return gst_element_get_base_time(element()); +} + +void QGstElement::setBaseTime(GstClockTime time) const +{ + gst_element_set_base_time(element(), time); +} + +GstElement *QGstElement::element() const +{ + return GST_ELEMENT_CAST(get()); +} + +QGstElement QGstElement::getParent() const +{ + return QGstElement{ + qGstCheckedCast<GstElement>(gst_element_get_parent(object())), + QGstElement::HasRef, + }; +} + +QGstPipeline QGstElement::getPipeline() const +{ + QGstElement ancestor = *this; + for (;;) { + QGstElement greatAncestor = ancestor.getParent(); + if (greatAncestor) { + ancestor = std::move(greatAncestor); + continue; + } + + return QGstPipeline{ + qGstSafeCast<GstPipeline>(ancestor.element()), + QGstPipeline::NeedsRef, + }; + } +} + +void QGstElement::dumpPipelineGraph(const char *filename) const +{ + static const bool dumpEnabled = qEnvironmentVariableIsSet("GST_DEBUG_DUMP_DOT_DIR"); + if (dumpEnabled) { + QGstPipeline pipeline = getPipeline(); + if (pipeline) + pipeline.dumpGraph(filename); + } +} + +QGstQueryHandle &QGstElement::positionQuery() const +{ + if (Q_UNLIKELY(!m_positionQuery)) + m_positionQuery = QGstQueryHandle{ + gst_query_new_position(GST_FORMAT_TIME), + QGstQueryHandle::HasRef, + }; + + return m_positionQuery; +} + +// QGstBin + +QGstBin QGstBin::create(const char *name) +{ + return QGstBin(gst_bin_new(name), NeedsRef); +} + +QGstBin QGstBin::createFromFactory(const char *factory, const char *name) +{ + QGstElement element = QGstElement::createFromFactory(factory, name); + Q_ASSERT(GST_IS_BIN(element.element())); + return QGstBin{ + GST_BIN(element.release()), + RefMode::HasRef, + }; +} + +QGstBin QGstBin::createFromPipelineDescription(const QByteArray &pipelineDescription, + const char *name, bool ghostUnlinkedPads) +{ + return createFromPipelineDescription(pipelineDescription.constData(), name, ghostUnlinkedPads); +} + +QGstBin QGstBin::createFromPipelineDescription(const char *pipelineDescription, const char *name, + bool ghostUnlinkedPads) +{ + QUniqueGErrorHandle error; + + GstElement *element = + gst_parse_bin_from_description_full(pipelineDescription, ghostUnlinkedPads, + /*context=*/nullptr, GST_PARSE_FLAG_NONE, &error); + + if (!element) { + qWarning() << "Failed to make element from pipeline description" << pipelineDescription + << error; + return QGstBin{}; + } + + if (name) + gst_element_set_name(element, name); + + return QGstBin{ + element, + NeedsRef, + }; +} + +QGstBin::QGstBin(GstBin *bin, RefMode mode) + : QGstElement{ + qGstCheckedCast<GstElement>(bin), + mode, + } +{ +} + +GstBin *QGstBin::bin() const +{ + return qGstCheckedCast<GstBin>(object()); +} + +void QGstBin::addGhostPad(const QGstElement &child, const char *name) +{ + addGhostPad(name, child.staticPad(name)); +} + +void QGstBin::addGhostPad(const char *name, const QGstPad &pad) +{ + gst_element_add_pad(element(), gst_ghost_pad_new(name, pad.pad())); +} + +bool QGstBin::syncChildrenState() +{ + return gst_bin_sync_children_states(bin()); +} + +void QGstBin::dumpGraph(const char *fileNamePrefix) const +{ + if (isNull()) + return; + + GST_DEBUG_BIN_TO_DOT_FILE(bin(), GST_DEBUG_GRAPH_SHOW_VERBOSE, fileNamePrefix); +} + +QGstElement QGstBin::findByName(const char *name) +{ + return QGstElement{ + gst_bin_get_by_name(bin(), name), + QGstElement::NeedsRef, + }; +} + +// QGstBaseSink + +QGstBaseSink::QGstBaseSink(GstBaseSink *element, RefMode mode) + : QGstElement{ + qGstCheckedCast<GstElement>(element), + mode, + } +{ +} + +void QGstBaseSink::setSync(bool arg) +{ + gst_base_sink_set_sync(baseSink(), arg ? TRUE : FALSE); +} + +GstBaseSink *QGstBaseSink::baseSink() const +{ + return qGstCheckedCast<GstBaseSink>(element()); +} + +// QGstBaseSrc + +QGstBaseSrc::QGstBaseSrc(GstBaseSrc *element, RefMode mode) + : QGstElement{ + qGstCheckedCast<GstElement>(element), + mode, + } +{ +} + +GstBaseSrc *QGstBaseSrc::baseSrc() const +{ + return qGstCheckedCast<GstBaseSrc>(element()); +} + +// QGstAppSink + +QGstAppSink::QGstAppSink(GstAppSink *element, RefMode mode) + : QGstBaseSink{ + qGstCheckedCast<GstBaseSink>(element), + mode, + } +{ +} + +QGstAppSink QGstAppSink::create(const char *name) +{ + QGstElement created = QGstElement::createFromFactory("appsink", name); + return QGstAppSink{ + qGstCheckedCast<GstAppSink>(created.element()), + QGstAppSink::NeedsRef, + }; +} + +GstAppSink *QGstAppSink::appSink() const +{ + return qGstCheckedCast<GstAppSink>(element()); +} + +# if GST_CHECK_VERSION(1, 24, 0) +void QGstAppSink::setMaxBufferTime(std::chrono::nanoseconds ns) +{ + gst_app_sink_set_max_time(appSink(), qGstClockTimeFromChrono(ns)); +} +# endif + +void QGstAppSink::setMaxBuffers(int n) +{ + gst_app_sink_set_max_buffers(appSink(), n); +} + +void QGstAppSink::setCaps(const QGstCaps &caps) +{ + gst_app_sink_set_caps(appSink(), caps.caps()); +} + +void QGstAppSink::setCallbacks(GstAppSinkCallbacks &callbacks, gpointer user_data, + GDestroyNotify notify) +{ + gst_app_sink_set_callbacks(appSink(), &callbacks, user_data, notify); +} + +QGstSampleHandle QGstAppSink::pullSample() +{ + return QGstSampleHandle{ + gst_app_sink_pull_sample(appSink()), + QGstSampleHandle::HasRef, + }; +} + +// QGstAppSrc + +QGstAppSrc::QGstAppSrc(GstAppSrc *element, RefMode mode) + : QGstBaseSrc{ + qGstCheckedCast<GstBaseSrc>(element), + mode, + } +{ +} + +QGstAppSrc QGstAppSrc::create(const char *name) +{ + QGstElement created = QGstElement::createFromFactory("appsrc", name); + return QGstAppSrc{ + qGstCheckedCast<GstAppSrc>(created.element()), + QGstAppSrc::NeedsRef, + }; +} + +GstAppSrc *QGstAppSrc::appSrc() const +{ + return qGstCheckedCast<GstAppSrc>(element()); +} + +void QGstAppSrc::setCallbacks(GstAppSrcCallbacks &callbacks, gpointer user_data, + GDestroyNotify notify) +{ + gst_app_src_set_callbacks(appSrc(), &callbacks, user_data, notify); +} + +GstFlowReturn QGstAppSrc::pushBuffer(GstBuffer *buffer) +{ + return gst_app_src_push_buffer(appSrc(), buffer); +} + +QString qGstErrorMessageCannotFindElement(std::string_view element) +{ + return QStringLiteral("Could not find the %1 GStreamer element") + .arg(QLatin1StringView(element)); +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp b/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp new file mode 100644 index 000000000..e47515d2d --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp @@ -0,0 +1,573 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qgst_debug_p.h" +#include "qgstreamermessage_p.h" + +#include <gst/gstclock.h> + +QT_BEGIN_NAMESPACE + +// NOLINTBEGIN(performance-unnecessary-value-param) + +QDebug operator<<(QDebug dbg, const QGString &str) +{ + return dbg << str.get(); +} + +QDebug operator<<(QDebug dbg, const QGstCaps &caps) +{ + return dbg << caps.caps(); +} + +QDebug operator<<(QDebug dbg, const QGstStructureView &structure) +{ + return dbg << structure.structure; +} + +QDebug operator<<(QDebug dbg, const QGValue &value) +{ + return dbg << value.value; +} + +QDebug operator<<(QDebug dbg, const QGstreamerMessage &msg) +{ + return dbg << msg.message(); +} + +QDebug operator<<(QDebug dbg, const QUniqueGErrorHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QUniqueGStringHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstStreamCollectionHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstStreamHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstTagListHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstElement &element) +{ + return dbg << element.element(); +} + +QDebug operator<<(QDebug dbg, const QGstPad &pad) +{ + return dbg << pad.pad(); +} + +QDebug operator<<(QDebug dbg, const GstCaps *caps) +{ + if (caps) + return dbg << QGString(gst_caps_to_string(caps)); + else + return dbg << "null"; +} + +QDebug operator<<(QDebug dbg, const GstVideoInfo *info) +{ +#if GST_CHECK_VERSION(1, 20, 0) + return dbg << QGstCaps{ + gst_video_info_to_caps(info), + QGstCaps::NeedsRef, + }; +#else + return dbg << QGstCaps{ + gst_video_info_to_caps(const_cast<GstVideoInfo *>(info)), + QGstCaps::NeedsRef, + }; +#endif +} + +QDebug operator<<(QDebug dbg, const GstStructure *structure) +{ + if (structure) + return dbg << QGString(gst_structure_to_string(structure)); + else + return dbg << "null"; +} + +QDebug operator<<(QDebug dbg, const GstObject *object) +{ + dbg << QGString{gst_object_get_name(const_cast<GstObject*>(object))}; + + { + QDebugStateSaver saver(dbg); + dbg.nospace(); + + dbg << "{"; + + guint numProperties; + GParamSpec **properties = g_object_class_list_properties(G_OBJECT_GET_CLASS(object), &numProperties); + + for (guint i = 0; i < numProperties; i++) { + GParamSpec *param = properties[i]; + + const gchar *name = g_param_spec_get_name(param); + constexpr bool trace_blurb = false; + if constexpr (trace_blurb) { + const gchar *blurb = g_param_spec_get_blurb(param); + dbg << name << " (" << blurb << "): "; + } else + dbg << name << ": "; + + bool readable = bool(param->flags & G_PARAM_READABLE); + if (!readable) { + dbg << "(not readable)"; + } else if (QLatin1StringView(name) == QLatin1StringView("parent")) { + if (object->parent) + dbg << QGString{ gst_object_get_name(object->parent) }; + else + dbg << "(none)"; + } else { + GValue value = {}; + g_object_get_property(&const_cast<GstObject *>(object)->object, param->name, + &value); + dbg << &value; + } + if (i != numProperties - 1) + dbg << ", "; + } + + dbg << "}"; + + g_free(properties); + } + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstElement *element) +{ + return dbg << GST_OBJECT_CAST(element); // LATER: output other members? +} + +QDebug operator<<(QDebug dbg, const GstPad *pad) +{ + return dbg << GST_OBJECT_CAST(pad); // LATER: output other members? +} + +QDebug operator<<(QDebug dbg, const GstDevice *device) +{ + GstDevice *d = const_cast<GstDevice *>(device); + QDebugStateSaver saver(dbg); + dbg.nospace(); + + dbg << gst_device_get_display_name(d) << "(" << gst_device_get_device_class(d) << ") "; + dbg << "Caps: " << QGstCaps{ gst_device_get_caps(d), QGstCaps::NeedsRef, } << ", "; + dbg << "Properties: " << QUniqueGstStructureHandle{ gst_device_get_properties(d) }.get(); + return dbg; +} + +namespace { + +struct Timepoint +{ + explicit Timepoint(guint64 us) : ts{ us } { } + guint64 ts; +}; + +QDebug operator<<(QDebug dbg, Timepoint ts) +{ + char buffer[128]; + snprintf(buffer, sizeof(buffer), "%" GST_TIME_FORMAT, GST_TIME_ARGS(ts.ts)); + dbg << buffer; + return dbg; +} + +} // namespace + +QDebug operator<<(QDebug dbg, const GstMessage *msg) +{ + QDebugStateSaver saver(dbg); + dbg.nospace(); + + dbg << GST_MESSAGE_TYPE_NAME(msg) << ", Source: " << GST_MESSAGE_SRC_NAME(msg); + if (GST_MESSAGE_TIMESTAMP(msg) != 0xFFFFFFFFFFFFFFFF) + dbg << ", Timestamp: " << GST_MESSAGE_TIMESTAMP(msg); + + switch (msg->type) { + case GST_MESSAGE_ERROR: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Error: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_WARNING: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_warning(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Warning: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_INFO: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_info(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Info: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_TAG: { + QGstTagListHandle tagList; + gst_message_parse_tag(const_cast<GstMessage *>(msg), &tagList); + + dbg << ", Tags: " << tagList; + break; + } + + case GST_MESSAGE_QOS: { + gboolean live; + guint64 running_time; + guint64 stream_time; + guint64 timestamp; + guint64 duration; + + gst_message_parse_qos(const_cast<GstMessage *>(msg), &live, &running_time, &stream_time, + ×tamp, &duration); + + dbg << ", Live: " << bool(live) << ", Running time: " << Timepoint{ running_time } + << ", Stream time: " << Timepoint{ stream_time } + << ", Timestamp: " << Timepoint{ timestamp } << ", Duration: " << Timepoint{ duration }; + break; + } + + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(const_cast<GstMessage *>(msg), &oldState, &newState, + &pending); + + dbg << ", Transition: " << oldState << "->" << newState; + + if (pending != GST_STATE_VOID_PENDING) + dbg << ", Pending State: " << pending; + break; + } + + case GST_MESSAGE_STREAM_COLLECTION: { + QGstStreamCollectionHandle collection; + gst_message_parse_stream_collection(const_cast<GstMessage *>(msg), &collection); + + dbg << ", " << collection; + break; + } + + case GST_MESSAGE_STREAMS_SELECTED: { + QGstStreamCollectionHandle collection; + gst_message_parse_streams_selected(const_cast<GstMessage *>(msg), &collection); + + dbg << ", " << collection; + break; + } + + case GST_MESSAGE_STREAM_STATUS: { + GstStreamStatusType streamStatus; + gst_message_parse_stream_status(const_cast<GstMessage *>(msg), &streamStatus, nullptr); + + dbg << ", Stream Status: " << streamStatus; + break; + } + + case GST_MESSAGE_BUFFERING: { + int progress = 0; + gst_message_parse_buffering(const_cast<GstMessage *>(msg), &progress); + + dbg << ", Buffering: " << progress << "%"; + break; + } + + default: + break; + } + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstTagList *tagList) +{ + dbg << QGString{ gst_tag_list_to_string(tagList) }; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstQuery *query) +{ + dbg << GST_QUERY_TYPE_NAME(query); + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstEvent *event) +{ + dbg << GST_EVENT_TYPE_NAME(event); + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstPadTemplate *padTemplate) +{ + QGstCaps caps = padTemplate + ? QGstCaps{ gst_pad_template_get_caps(const_cast<GstPadTemplate *>(padTemplate)), QGstCaps::HasRef, } + : QGstCaps{}; + + dbg << caps; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstStreamCollection *streamCollection) +{ + GstStreamCollection *collection = const_cast<GstStreamCollection *>(streamCollection); + guint size = gst_stream_collection_get_size(collection); + + dbg << "Stream Collection: {"; + for (guint index = 0; index != size; ++index) { + dbg << gst_stream_collection_get_stream(collection, index); + if (index + 1 != size) + dbg << ", "; + } + + dbg << "}"; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstStream *cstream) +{ + GstStream *stream = const_cast<GstStream *>(cstream); + + dbg << "GstStream { "; + dbg << "Type: " << gst_stream_type_get_name(gst_stream_get_stream_type(stream)); + + QGstTagListHandle tagList{ + gst_stream_get_tags(stream), + QGstTagListHandle::HasRef, + }; + + if (tagList) + dbg << ", Tags: " << tagList; + + QGstCaps caps{ + gst_stream_get_caps(stream), + QGstCaps::HasRef, + }; + + if (caps) + dbg << ", Caps: " << caps; + + dbg << "}"; + + return dbg; +} + +QDebug operator<<(QDebug dbg, GstState state) +{ + return dbg << gst_element_state_get_name(state); +} + +QDebug operator<<(QDebug dbg, GstStateChange transition) +{ + return dbg << gst_state_change_get_name(transition); +} + +QDebug operator<<(QDebug dbg, GstStateChangeReturn stateChangeReturn) +{ + return dbg << gst_element_state_change_return_get_name(stateChangeReturn); +} + +QDebug operator<<(QDebug dbg, GstMessageType type) +{ + return dbg << gst_message_type_get_name(type); +} + +#define ADD_ENUM_SWITCH(value) \ + case value: \ + return dbg << #value; \ + static_assert(true, "enforce semicolon") + +QDebug operator<<(QDebug dbg, GstPadDirection direction) +{ + switch (direction) { + ADD_ENUM_SWITCH(GST_PAD_UNKNOWN); + ADD_ENUM_SWITCH(GST_PAD_SRC); + ADD_ENUM_SWITCH(GST_PAD_SINK); + default: + Q_UNREACHABLE_RETURN(dbg); + } +} + +QDebug operator<<(QDebug dbg, GstStreamStatusType type) +{ + switch (type) { + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_CREATE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_ENTER); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_LEAVE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_DESTROY); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_START); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_PAUSE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_STOP); + default: + Q_UNREACHABLE_RETURN(dbg); + } + return dbg; +} + +#undef ADD_ENUM_SWITCH + +QDebug operator<<(QDebug dbg, const GValue *value) +{ + switch (G_VALUE_TYPE(value)) { + case G_TYPE_STRING: + return dbg << g_value_get_string(value); + case G_TYPE_BOOLEAN: + return dbg << g_value_get_boolean(value); + case G_TYPE_ULONG: + return dbg << g_value_get_ulong(value); + case G_TYPE_LONG: + return dbg << g_value_get_long(value); + case G_TYPE_UINT: + return dbg << g_value_get_uint(value); + case G_TYPE_INT: + return dbg << g_value_get_int(value); + case G_TYPE_UINT64: + return dbg << g_value_get_uint64(value); + case G_TYPE_INT64: + return dbg << g_value_get_int64(value); + case G_TYPE_FLOAT: + return dbg << g_value_get_float(value); + case G_TYPE_DOUBLE: + return dbg << g_value_get_double(value); + default: + break; + } + + if (GST_VALUE_HOLDS_BITMASK(value)) { + QDebugStateSaver saver(dbg); + return dbg << Qt::hex << gst_value_get_bitmask(value); + } + + if (GST_VALUE_HOLDS_FRACTION(value)) + return dbg << gst_value_get_fraction_numerator(value) << "/" + << gst_value_get_fraction_denominator(value); + + if (GST_VALUE_HOLDS_CAPS(value)) + return dbg << gst_value_get_caps(value); + + if (GST_VALUE_HOLDS_STRUCTURE(value)) + return dbg << gst_value_get_structure(value); + + if (GST_VALUE_HOLDS_ARRAY(value)) { + const guint size = gst_value_array_get_size(value); + const guint last = size - 1; + dbg << "["; + for (guint index = 0; index != size; ++index) { + dbg << gst_value_array_get_value(value, index); + if (index != last) + dbg << ", "; + } + dbg << "}"; + return dbg; + } + + if (G_VALUE_TYPE(value) == GST_TYPE_PAD_DIRECTION) { + GstPadDirection direction = static_cast<GstPadDirection>(g_value_get_enum(value)); + return dbg << direction; + } + + if (G_VALUE_TYPE(value) == GST_TYPE_PAD_TEMPLATE) { + GstPadTemplate *padTemplate = static_cast<GstPadTemplate *>(g_value_get_object(value)); + return dbg << padTemplate; + } + + dbg << "(not implemented: " << G_VALUE_TYPE_NAME(value) << ")"; + + return dbg; +} + +QDebug operator<<(QDebug dbg, const GError *error) +{ + return dbg << error->message; +} + +QCompactGstMessageAdaptor::QCompactGstMessageAdaptor(const QGstreamerMessage &m) + : QCompactGstMessageAdaptor{ + m.message(), + } +{ +} + +QCompactGstMessageAdaptor::QCompactGstMessageAdaptor(GstMessage *m) + : msg{ + m, + } +{ +} + +QDebug operator<<(QDebug dbg, const QCompactGstMessageAdaptor &m) +{ + std::optional<QDebugStateSaver> saver(dbg); + dbg.nospace(); + + switch (GST_MESSAGE_TYPE(m.msg)) { + case GST_MESSAGE_ERROR: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(m.msg, &err, &debug); + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_WARNING: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_warning(m.msg, &err, &debug); + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_INFO: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_info(m.msg, &err, &debug); + + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(m.msg, &oldState, &newState, &pending); + + dbg << oldState << " -> " << newState; + if (pending != GST_STATE_VOID_PENDING) + dbg << " (pending: " << pending << ")"; + return dbg; + } + + default: { + saver.reset(); + return dbg << m.msg; + } + } +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h b/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h new file mode 100644 index 000000000..df13c6c13 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h @@ -0,0 +1,74 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGST_DEBUG_P_H +#define QGST_DEBUG_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qgst_p.h" +#include <qdebug.h> + +QT_BEGIN_NAMESPACE + +class QGstreamerMessage; + +QDebug operator<<(QDebug, const QGstCaps &); +QDebug operator<<(QDebug, const QGstStructureView &); +QDebug operator<<(QDebug, const QGstElement &); +QDebug operator<<(QDebug, const QGstPad &); +QDebug operator<<(QDebug, const QGString &); +QDebug operator<<(QDebug, const QGValue &); +QDebug operator<<(QDebug, const QGstreamerMessage &); +QDebug operator<<(QDebug, const QUniqueGErrorHandle &); +QDebug operator<<(QDebug, const QUniqueGStringHandle &); +QDebug operator<<(QDebug, const QGstStreamCollectionHandle &); +QDebug operator<<(QDebug, const QGstStreamHandle &); +QDebug operator<<(QDebug, const QGstTagListHandle &); + +QDebug operator<<(QDebug, const GstCaps *); +QDebug operator<<(QDebug, const GstVideoInfo *); +QDebug operator<<(QDebug, const GstStructure *); +QDebug operator<<(QDebug, const GstObject *); +QDebug operator<<(QDebug, const GstElement *); +QDebug operator<<(QDebug, const GstPad *); +QDebug operator<<(QDebug, const GstDevice *); +QDebug operator<<(QDebug, const GstMessage *); +QDebug operator<<(QDebug, const GstTagList *); +QDebug operator<<(QDebug, const GstQuery *); +QDebug operator<<(QDebug, const GstEvent *); +QDebug operator<<(QDebug, const GstPadTemplate *); +QDebug operator<<(QDebug, const GstStreamCollection *); +QDebug operator<<(QDebug, const GstStream *); + +QDebug operator<<(QDebug, GstState); +QDebug operator<<(QDebug, GstStateChange); +QDebug operator<<(QDebug, GstStateChangeReturn); +QDebug operator<<(QDebug, GstMessageType); +QDebug operator<<(QDebug, GstPadDirection); +QDebug operator<<(QDebug, GstStreamStatusType); + +QDebug operator<<(QDebug, const GValue *); +QDebug operator<<(QDebug, const GError *); + +struct QCompactGstMessageAdaptor +{ + explicit QCompactGstMessageAdaptor(const QGstreamerMessage &m); + explicit QCompactGstMessageAdaptor(GstMessage *m); + GstMessage *msg; +}; + +QDebug operator<<(QDebug, const QCompactGstMessageAdaptor &); + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h b/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h new file mode 100644 index 000000000..e813f4181 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h @@ -0,0 +1,270 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGST_HANDLE_TYPES_P_H +#define QGST_HANDLE_TYPES_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/private/qcore_unix_p.h> +#include <QtCore/private/quniquehandle_p.h> +#include <QtCore/qtconfigmacros.h> + +#include <QtMultimedia/private/qtmultimedia-config_p.h> + +#include <gst/gst.h> + +#if QT_CONFIG(gstreamer_gl) +# include <gst/gl/gstglcontext.h> +#endif + +QT_BEGIN_NAMESPACE + +namespace QGstImpl { + +template <typename HandleTraits> +struct QSharedHandle : private QUniqueHandle<HandleTraits> +{ + using BaseClass = QUniqueHandle<HandleTraits>; + + enum RefMode { HasRef, NeedsRef }; + + QSharedHandle() = default; + + explicit QSharedHandle(typename HandleTraits::Type object, RefMode mode) + : BaseClass{ mode == NeedsRef ? HandleTraits::ref(object) : object } + { + } + + QSharedHandle(const QSharedHandle &o) + : BaseClass{ + HandleTraits::ref(o.get()), + } + { + } + + QSharedHandle(QSharedHandle &&) noexcept = default; + + QSharedHandle &operator=(const QSharedHandle &o) // NOLINT: bugprone-unhandled-self-assign + { + if (BaseClass::get() != o.get()) + reset(HandleTraits::ref(o.get())); + return *this; + }; + + QSharedHandle &operator=(QSharedHandle &&) noexcept = default; + + [[nodiscard]] friend bool operator==(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() == rhs.get(); + } + + [[nodiscard]] friend bool operator!=(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() != rhs.get(); + } + + [[nodiscard]] friend bool operator<(const QSharedHandle &lhs, const QSharedHandle &rhs) noexcept + { + return lhs.get() < rhs.get(); + } + + [[nodiscard]] friend bool operator<=(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() <= rhs.get(); + } + + [[nodiscard]] friend bool operator>(const QSharedHandle &lhs, const QSharedHandle &rhs) noexcept + { + return lhs.get() > rhs.get(); + } + + [[nodiscard]] friend bool operator>=(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() >= rhs.get(); + } + + using BaseClass::get; + using BaseClass::isValid; + using BaseClass::operator bool; + using BaseClass::release; + using BaseClass::reset; + using BaseClass::operator&; + using BaseClass::close; +}; + +struct QGstTagListHandleTraits +{ + using Type = GstTagList *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_tag_list_unref(handle); + return true; + } + static Type ref(Type handle) noexcept { return gst_tag_list_ref(handle); } +}; + +struct QGstSampleHandleTraits +{ + using Type = GstSample *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_sample_unref(handle); + return true; + } + static Type ref(Type handle) noexcept { return gst_sample_ref(handle); } +}; + +struct QUniqueGstStructureHandleTraits +{ + using Type = GstStructure *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_structure_free(handle); + return true; + } +}; + +struct QUniqueGStringHandleTraits +{ + using Type = gchar *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + g_free(handle); + return true; + } +}; + +struct QUniqueGErrorHandleTraits +{ + using Type = GError *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + g_error_free(handle); + return true; + } +}; + + +struct QUniqueGstDateTimeHandleTraits +{ + using Type = GstDateTime *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_date_time_unref(handle); + return true; + } +}; + +struct QFileDescriptorHandleTraits +{ + using Type = int; + static constexpr Type invalidValue() noexcept { return -1; } + static bool close(Type fd) noexcept + { + int closeResult = qt_safe_close(fd); + return closeResult == 0; + } +}; + +template <typename GstType> +struct QGstHandleHelper +{ + struct QGstSafeObjectHandleTraits + { + using Type = GstType *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_object_unref(G_OBJECT(handle)); + return true; + } + + static Type ref(Type handle) noexcept + { + gst_object_ref_sink(G_OBJECT(handle)); + return handle; + } + }; + + using SharedHandle = QSharedHandle<QGstSafeObjectHandleTraits>; + using UniqueHandle = QUniqueHandle<QGstSafeObjectHandleTraits>; +}; + +template <typename GstType> +struct QGstMiniObjectHandleHelper +{ + struct Traits + { + using Type = GstType *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_mini_object_unref(GST_MINI_OBJECT_CAST(handle)); + return true; + } + + static Type ref(Type handle) noexcept + { + if (GST_MINI_OBJECT_CAST(handle)) + gst_mini_object_ref(GST_MINI_OBJECT_CAST(handle)); + return handle; + } + }; + + using SharedHandle = QSharedHandle<Traits>; + using UniqueHandle = QUniqueHandle<Traits>; +}; + +} // namespace QGstImpl + +using QGstClockHandle = QGstImpl::QGstHandleHelper<GstClock>::UniqueHandle; +using QGstElementHandle = QGstImpl::QGstHandleHelper<GstElement>::UniqueHandle; +using QGstElementFactoryHandle = QGstImpl::QGstHandleHelper<GstElementFactory>::SharedHandle; +using QGstDeviceHandle = QGstImpl::QGstHandleHelper<GstDevice>::SharedHandle; +using QGstDeviceMonitorHandle = QGstImpl::QGstHandleHelper<GstDeviceMonitor>::UniqueHandle; +using QGstBusHandle = QGstImpl::QGstHandleHelper<GstBus>::UniqueHandle; +using QGstStreamCollectionHandle = QGstImpl::QGstHandleHelper<GstStreamCollection>::SharedHandle; +using QGstStreamHandle = QGstImpl::QGstHandleHelper<GstStream>::SharedHandle; + +using QGstTagListHandle = QGstImpl::QSharedHandle<QGstImpl::QGstTagListHandleTraits>; +using QGstSampleHandle = QGstImpl::QSharedHandle<QGstImpl::QGstSampleHandleTraits>; + +using QUniqueGstStructureHandle = QUniqueHandle<QGstImpl::QUniqueGstStructureHandleTraits>; +using QUniqueGStringHandle = QUniqueHandle<QGstImpl::QUniqueGStringHandleTraits>; +using QUniqueGErrorHandle = QUniqueHandle<QGstImpl::QUniqueGErrorHandleTraits>; +using QUniqueGstDateTimeHandle = QUniqueHandle<QGstImpl::QUniqueGstDateTimeHandleTraits>; +using QFileDescriptorHandle = QUniqueHandle<QGstImpl::QFileDescriptorHandleTraits>; +using QGstBufferHandle = QGstImpl::QGstMiniObjectHandleHelper<GstBuffer>::SharedHandle; +using QGstContextHandle = QGstImpl::QGstMiniObjectHandleHelper<GstContext>::UniqueHandle; +using QGstGstDateTimeHandle = QGstImpl::QGstMiniObjectHandleHelper<GstDateTime>::SharedHandle; +using QGstPluginFeatureHandle = QGstImpl::QGstHandleHelper<GstPluginFeature>::SharedHandle; +using QGstQueryHandle = QGstImpl::QGstMiniObjectHandleHelper<GstQuery>::SharedHandle; + +#if QT_CONFIG(gstreamer_gl) +using QGstGLContextHandle = QGstImpl::QGstHandleHelper<GstGLContext>::UniqueHandle; +using QGstGLDisplayHandle = QGstImpl::QGstHandleHelper<GstGLDisplay>::UniqueHandle; +#endif + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst_p.h b/src/plugins/multimedia/gstreamer/common/qgst_p.h index 66b1b156f..bf5290d5d 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgst_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGST_P_H #define QGST_P_H @@ -51,32 +15,159 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> - -#include <QSemaphore> +#include <QtCore/qdebug.h> #include <QtCore/qlist.h> +#include <QtCore/qsemaphore.h> #include <QtMultimedia/qaudioformat.h> #include <QtMultimedia/qvideoframe.h> +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtMultimedia/private/qmultimediautils_p.h> +#include <QtMultimedia/private/qplatformmediaplayer_p.h> #include <gst/gst.h> +#include <gst/app/gstappsink.h> +#include <gst/app/gstappsrc.h> #include <gst/video/video-info.h> -#include <functional> +#include "qgst_handle_types_p.h" + +#include <type_traits> #if QT_CONFIG(gstreamer_photography) -#define GST_USE_UNSTABLE_API -#include <gst/interfaces/photography.h> -#undef GST_USE_UNSTABLE_API -#endif -#ifndef QT_NO_DEBUG -#include <qdebug.h> +# define GST_USE_UNSTABLE_API +# include <gst/interfaces/photography.h> +# undef GST_USE_UNSTABLE_API #endif + QT_BEGIN_NAMESPACE +namespace QGstImpl { + +template <typename T> +struct GstObjectTraits +{ + // using Type = T; + // template <typename U> + // static bool isObjectOfType(U *); + // template <typename U> + // static T *cast(U *); +}; + +#define QGST_DEFINE_CAST_TRAITS(ClassName, MACRO_LABEL) \ + template <> \ + struct GstObjectTraits<ClassName> \ + { \ + using Type = ClassName; \ + template <typename U> \ + static bool isObjectOfType(U *arg) \ + { \ + return GST_IS_##MACRO_LABEL(arg); \ + } \ + template <typename U> \ + static Type *cast(U *arg) \ + { \ + return GST_##MACRO_LABEL##_CAST(arg); \ + } \ + template <typename U> \ + static Type *checked_cast(U *arg) \ + { \ + return GST_##MACRO_LABEL(arg); \ + } \ + }; \ + static_assert(true, "ensure semicolon") + +#define QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE(ClassName, MACRO_LABEL) \ + template <> \ + struct GstObjectTraits<ClassName> \ + { \ + using Type = ClassName; \ + template <typename U> \ + static bool isObjectOfType(U *arg) \ + { \ + return GST_IS_##MACRO_LABEL(arg); \ + } \ + template <typename U> \ + static Type *cast(U *arg) \ + { \ + return checked_cast(arg); \ + } \ + template <typename U> \ + static Type *checked_cast(U *arg) \ + { \ + return GST_##MACRO_LABEL(arg); \ + } \ + }; \ + static_assert(true, "ensure semicolon") + +QGST_DEFINE_CAST_TRAITS(GstBin, BIN); +QGST_DEFINE_CAST_TRAITS(GstClock, CLOCK); +QGST_DEFINE_CAST_TRAITS(GstElement, ELEMENT); +QGST_DEFINE_CAST_TRAITS(GstObject, OBJECT); +QGST_DEFINE_CAST_TRAITS(GstPad, PAD); +QGST_DEFINE_CAST_TRAITS(GstPipeline, PIPELINE); +QGST_DEFINE_CAST_TRAITS(GstBaseSink, BASE_SINK); +QGST_DEFINE_CAST_TRAITS(GstBaseSrc, BASE_SRC); +QGST_DEFINE_CAST_TRAITS(GstAppSink, APP_SINK); +QGST_DEFINE_CAST_TRAITS(GstAppSrc, APP_SRC); + +QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE(GstTagSetter, TAG_SETTER); + + +template <> +struct GstObjectTraits<GObject> +{ + using Type = GObject; + template <typename U> + static bool isObjectOfType(U *arg) + { + return G_IS_OBJECT(arg); + } + template <typename U> + static Type *cast(U *arg) + { + return G_OBJECT(arg); + } + template <typename U> + static Type *checked_cast(U *arg) + { + return G_OBJECT(arg); + } +}; + +#undef QGST_DEFINE_CAST_TRAITS +#undef QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE + +} // namespace QGstImpl + +template <typename DestinationType, typename SourceType> +bool qIsGstObjectOfType(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + return arg && Traits::isObjectOfType(arg); +} + +template <typename DestinationType, typename SourceType> +DestinationType *qGstSafeCast(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + if (arg && Traits::isObjectOfType(arg)) + return Traits::cast(arg); + return nullptr; +} + +template <typename DestinationType, typename SourceType> +DestinationType *qGstCheckedCast(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + if (arg) + Q_ASSERT(Traits::isObjectOfType(arg)); + return Traits::cast(arg); +} + class QSize; -class QGstStructure; +class QGstStructureView; class QGstCaps; class QGstPipelinePrivate; class QCameraFormat; @@ -87,265 +178,284 @@ template <typename T> struct QGRange T max; }; -class QGString +struct QGString : QUniqueGStringHandle { - char *str; -public: - QGString(char *string) : str(string) {} - ~QGString() { g_free(str); } - operator QByteArray() { return QByteArray(str); } - operator const char *() { return str; } + using QUniqueGStringHandle::QUniqueGStringHandle; + + QLatin1StringView asStringView() const { return QLatin1StringView{ get() }; } + QString toQString() const { return QString::fromUtf8(get()); } }; class QGValue { public: - QGValue(const GValue *v) : value(v) {} + explicit QGValue(const GValue *v); const GValue *value; - bool isNull() const { return !value; } + bool isNull() const; - std::optional<bool> toBool() const + std::optional<bool> toBool() const; + std::optional<int> toInt() const; + std::optional<int> toInt64() const; + template<typename T> + T *getPointer() const { - if (!G_VALUE_HOLDS_BOOLEAN(value)) - return std::nullopt; - return g_value_get_boolean(value); + return value ? static_cast<T *>(g_value_get_pointer(value)) : nullptr; } - std::optional<int> toInt() const + + const char *toString() const; + std::optional<float> getFraction() const; + std::optional<QGRange<float>> getFractionRange() const; + std::optional<QGRange<int>> toIntRange() const; + + QGstStructureView toStructure() const; + QGstCaps toCaps() const; + + bool isList() const; + int listSize() const; + QGValue at(int index) const; + + QList<QAudioFormat::SampleFormat> getSampleFormats() const; +}; + +namespace QGstPointerImpl { + +template <typename RefcountedObject> +struct QGstRefcountingAdaptor; + +template <typename GstType> +class QGstObjectWrapper +{ + using Adaptor = QGstRefcountingAdaptor<GstType>; + + GstType *m_object = nullptr; + +public: + enum RefMode { HasRef, NeedsRef }; + + constexpr QGstObjectWrapper() = default; + + explicit QGstObjectWrapper(GstType *object, RefMode mode) : m_object(object) { - if (!G_VALUE_HOLDS_INT(value)) - return std::nullopt; - return g_value_get_int(value); + if (m_object && mode == NeedsRef) + Adaptor::ref(m_object); } - std::optional<int> toInt64() const + + QGstObjectWrapper(const QGstObjectWrapper &other) : m_object(other.m_object) { - if (!G_VALUE_HOLDS_INT64(value)) - return std::nullopt; - return g_value_get_int64(value); + if (m_object) + Adaptor::ref(m_object); } - template<typename T> - T *getPointer() const + + ~QGstObjectWrapper() { - return value ? static_cast<T *>(g_value_get_pointer(value)) : nullptr; + if (m_object) + Adaptor::unref(m_object); } - const char *toString() const + QGstObjectWrapper(QGstObjectWrapper &&other) noexcept + : m_object(std::exchange(other.m_object, nullptr)) { - return value ? g_value_get_string(value) : nullptr; } - std::optional<float> getFraction() const + + QGstObjectWrapper & + operator=(const QGstObjectWrapper &other) // NOLINT: bugprone-unhandled-self-assign { - if (!GST_VALUE_HOLDS_FRACTION(value)) - return std::nullopt; - return (float)gst_value_get_fraction_numerator(value)/(float)gst_value_get_fraction_denominator(value); + if (m_object != other.m_object) { + GstType *originalObject = m_object; + + m_object = other.m_object; + if (m_object) + Adaptor::ref(m_object); + if (originalObject) + Adaptor::unref(originalObject); + } + return *this; } - std::optional<QGRange<float>> getFractionRange() const + QGstObjectWrapper &operator=(QGstObjectWrapper &&other) noexcept { - if (!GST_VALUE_HOLDS_FRACTION_RANGE(value)) - return std::nullopt; - QGValue min = gst_value_get_fraction_range_min(value); - QGValue max = gst_value_get_fraction_range_max(value); - return QGRange<float>{ *min.getFraction(), *max.getFraction() }; + if (this != &other) { + GstType *originalObject = m_object; + m_object = std::exchange(other.m_object, nullptr); + + if (originalObject) + Adaptor::unref(originalObject); + } + return *this; } - std::optional<QGRange<int>> toIntRange() const + friend bool operator==(const QGstObjectWrapper &a, const QGstObjectWrapper &b) { - if (!GST_VALUE_HOLDS_INT_RANGE(value)) - return std::nullopt; - return QGRange<int>{ gst_value_get_int_range_min(value), gst_value_get_int_range_max(value) }; + return a.m_object == b.m_object; + } + friend bool operator!=(const QGstObjectWrapper &a, const QGstObjectWrapper &b) + { + return a.m_object != b.m_object; } - inline QGstStructure toStructure() const; - inline QGstCaps toCaps() const; - - inline bool isList() const { return value && GST_VALUE_HOLDS_LIST(value); } - inline int listSize() const { return gst_value_list_get_size(value); } - inline QGValue at(int index) const { return gst_value_list_get_value(value, index); } + explicit operator bool() const { return bool(m_object); } + bool isNull() const { return !m_object; } + GstType *release() { return std::exchange(m_object, nullptr); } - Q_MULTIMEDIA_EXPORT QList<QAudioFormat::SampleFormat> getSampleFormats() const; +protected: + GstType *get() const { return m_object; } }; -class QGstStructure { +} // namespace QGstPointerImpl + +class QGstreamerMessage; + +class QGstStructureView +{ public: const GstStructure *structure = nullptr; - QGstStructure() = default; - QGstStructure(const GstStructure *s) : structure(s) {} - void free() { if (structure) gst_structure_free(const_cast<GstStructure *>(structure)); structure = nullptr; } + explicit QGstStructureView(const GstStructure *); + explicit QGstStructureView(const QUniqueGstStructureHandle &); - bool isNull() const { return !structure; } + QUniqueGstStructureHandle clone() const; - QByteArrayView name() const { return gst_structure_get_name(structure); } + bool isNull() const; + QByteArrayView name() const; + QGValue operator[](const char *fieldname) const; - QGValue operator[](const char *name) const { return gst_structure_get_value(structure, name); } + QGstCaps caps() const; + QGstTagListHandle tags() const; - Q_MULTIMEDIA_EXPORT QSize resolution() const; - Q_MULTIMEDIA_EXPORT QVideoFrameFormat::PixelFormat pixelFormat() const; - Q_MULTIMEDIA_EXPORT QGRange<float> frameRateRange() const; + QSize resolution() const; + QVideoFrameFormat::PixelFormat pixelFormat() const; + QGRange<float> frameRateRange() const; + QGstreamerMessage getMessage(); + std::optional<Fraction> pixelAspectRatio() const; + QSize nativeSize() const; +}; - QByteArray toString() const - { - char *s = gst_structure_to_string(structure); - QByteArray str(s); - g_free(s); - return str; - } - QGstStructure copy() const { return gst_structure_copy(structure); } +template <> +struct QGstPointerImpl::QGstRefcountingAdaptor<GstCaps> +{ + static void ref(GstCaps *arg) noexcept { gst_caps_ref(arg); } + static void unref(GstCaps *arg) noexcept { gst_caps_unref(arg); } }; -class QGstCaps { - const GstCaps *caps = nullptr; +class QGstCaps : public QGstPointerImpl::QGstObjectWrapper<GstCaps> +{ + using BaseClass = QGstPointerImpl::QGstObjectWrapper<GstCaps>; + public: - enum MemoryFormat { - CpuMemory, - GLTexture, - DMABuf - }; + using BaseClass::BaseClass; + QGstCaps(const QGstCaps &) = default; + QGstCaps(QGstCaps &&) noexcept = default; + QGstCaps &operator=(const QGstCaps &) = default; + QGstCaps &operator=(QGstCaps &&) noexcept = default; - QGstCaps() = default; - QGstCaps(const GstCaps *c) : caps(c) {} + enum MemoryFormat { CpuMemory, GLTexture, DMABuf }; - bool isNull() const { return !caps; } + int size() const; + QGstStructureView at(int index) const; + GstCaps *caps() const; - int size() const { return gst_caps_get_size(caps); } - QGstStructure at(int index) { return gst_caps_get_structure(caps, index); } - const GstCaps *get() const { return caps; } - QByteArray toString() const - { - gchar *c = gst_caps_to_string(caps); - QByteArray b(c); - g_free(c); - return b; - } - MemoryFormat memoryFormat() const { - auto *features = gst_caps_get_features(caps, 0); - if (gst_caps_features_contains(features, "memory:GLMemory")) - return GLTexture; - else if (gst_caps_features_contains(features, "memory:DMABuf")) - return DMABuf; - return CpuMemory; - } - QVideoFrameFormat formatForCaps(GstVideoInfo *info) const; + MemoryFormat memoryFormat() const; + std::optional<std::pair<QVideoFrameFormat, GstVideoInfo>> formatAndVideoInfo() const; + + void addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, const char *modifier = nullptr); + void setResolution(QSize); + + static QGstCaps create(); + + static QGstCaps fromCameraFormat(const QCameraFormat &format); + + QGstCaps copy() const; +}; + +template <> +struct QGstPointerImpl::QGstRefcountingAdaptor<GstObject> +{ + static void ref(GstObject *arg) noexcept { gst_object_ref_sink(arg); } + static void unref(GstObject *arg) noexcept { gst_object_unref(arg); } }; -class QGstMutableCaps : public QGstCaps { - GstCaps *caps = nullptr; +class QGObjectHandlerConnection; + +class QGstObject : public QGstPointerImpl::QGstObjectWrapper<GstObject> +{ + using BaseClass = QGstPointerImpl::QGstObjectWrapper<GstObject>; + public: - enum RefMode { HasRef, NeedsRef }; - QGstMutableCaps() = default; - QGstMutableCaps(GstCaps *c, RefMode mode = HasRef) - : QGstCaps(c), caps(c) - { - Q_ASSERT(QGstCaps::get() == caps); - if (mode == NeedsRef) - gst_caps_ref(caps); - } - QGstMutableCaps(const QGstMutableCaps &other) - : QGstCaps(other), caps(other.caps) - { - Q_ASSERT(QGstCaps::get() == caps); - if (caps) - gst_caps_ref(caps); - } - QGstMutableCaps &operator=(const QGstMutableCaps &other) - { - QGstCaps::operator=(other); - if (other.caps) - gst_caps_ref(other.caps); - if (caps) - gst_caps_unref(caps); - caps = other.caps; - Q_ASSERT(QGstCaps::get() == caps); - return *this; - } - ~QGstMutableCaps() { - Q_ASSERT(QGstCaps::get() == caps); - if (caps) - gst_caps_unref(caps); - } + using BaseClass::BaseClass; + QGstObject(const QGstObject &) = default; + QGstObject(QGstObject &&) noexcept = default; + + QGstObject &operator=(const QGstObject &) = default; + QGstObject &operator=(QGstObject &&) noexcept = default; + + void set(const char *property, const char *str); + void set(const char *property, bool b); + void set(const char *property, uint i); + void set(const char *property, int i); + void set(const char *property, qint64 i); + void set(const char *property, quint64 i); + void set(const char *property, double d); + void set(const char *property, const QGstObject &o); + void set(const char *property, const QGstCaps &c); + + QGString getString(const char *property) const; + QGstStructureView getStructure(const char *property) const; + bool getBool(const char *property) const; + uint getUInt(const char *property) const; + int getInt(const char *property) const; + quint64 getUInt64(const char *property) const; + qint64 getInt64(const char *property) const; + float getFloat(const char *property) const; + double getDouble(const char *property) const; + QGstObject getObject(const char *property) const; + + QGObjectHandlerConnection connect(const char *name, GCallback callback, gpointer userData); + void disconnect(gulong handlerId); + + GType type() const; + QLatin1StringView typeName() const; + GstObject *object() const; + QLatin1StringView name() const; +}; - void create() { - caps = gst_caps_new_empty(); - QGstCaps::operator=(QGstCaps(caps)); - } +class QGObjectHandlerConnection +{ +public: + QGObjectHandlerConnection(QGstObject object, gulong handler); - void addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, const char *modifier = nullptr); - static QGstMutableCaps fromCameraFormat(const QCameraFormat &format); + QGObjectHandlerConnection() = default; + QGObjectHandlerConnection(const QGObjectHandlerConnection &) = default; + QGObjectHandlerConnection(QGObjectHandlerConnection &&) = default; + QGObjectHandlerConnection &operator=(const QGObjectHandlerConnection &) = default; + QGObjectHandlerConnection &operator=(QGObjectHandlerConnection &&) = default; + + void disconnect(); + +private: + static constexpr gulong invalidHandlerId = std::numeric_limits<gulong>::max(); - GstCaps *get() const { return caps; } + QGstObject object; + gulong handlerId = invalidHandlerId; }; -class QGstObject +// disconnects in dtor +class QGObjectHandlerScopedConnection { -protected: - GstObject *m_object = nullptr; public: - enum RefMode { HasRef, NeedsRef }; + QGObjectHandlerScopedConnection(QGObjectHandlerConnection connection); - QGstObject() = default; - explicit QGstObject(GstObject *o, RefMode mode = HasRef) - : m_object(o) - { - if (o && mode == NeedsRef) - // Use ref_sink to remove any floating references - gst_object_ref_sink(m_object); - } - QGstObject(const QGstObject &other) - : m_object(other.m_object) - { - if (m_object) - gst_object_ref(m_object); - } - QGstObject &operator=(const QGstObject &other) - { - if (this == &other) - return *this; - if (other.m_object) - gst_object_ref(other.m_object); - if (m_object) - gst_object_unref(m_object); - m_object = other.m_object; - return *this; - } - virtual ~QGstObject() { - if (m_object) - gst_object_unref(m_object); - } + QGObjectHandlerScopedConnection() = default; + QGObjectHandlerScopedConnection(const QGObjectHandlerScopedConnection &) = delete; + QGObjectHandlerScopedConnection &operator=(const QGObjectHandlerScopedConnection &) = delete; + QGObjectHandlerScopedConnection(QGObjectHandlerScopedConnection &&) = default; + QGObjectHandlerScopedConnection &operator=(QGObjectHandlerScopedConnection &&) = default; - friend bool operator==(const QGstObject &a, const QGstObject &b) - { return a.m_object == b.m_object; } - friend bool operator!=(const QGstObject &a, const QGstObject &b) - { return a.m_object != b.m_object; } + ~QGObjectHandlerScopedConnection(); - bool isNull() const { return !m_object; } + void disconnect(); - void set(const char *property, const char *str) { g_object_set(m_object, property, str, nullptr); } - void set(const char *property, bool b) { g_object_set(m_object, property, gboolean(b), nullptr); } - void set(const char *property, uint i) { g_object_set(m_object, property, guint(i), nullptr); } - void set(const char *property, int i) { g_object_set(m_object, property, gint(i), nullptr); } - void set(const char *property, qint64 i) { g_object_set(m_object, property, gint64(i), nullptr); } - void set(const char *property, quint64 i) { g_object_set(m_object, property, guint64(i), nullptr); } - void set(const char *property, double d) { g_object_set(m_object, property, gdouble(d), nullptr); } - void set(const char *property, const QGstObject &o) { g_object_set(m_object, property, o.object(), nullptr); } - void set(const char *property, const QGstMutableCaps &c) { g_object_set(m_object, property, c.get(), nullptr); } - - QGString getString(const char *property) const - { char *s = nullptr; g_object_get(m_object, property, &s, nullptr); return s; } - QGstStructure getStructure(const char *property) const - { GstStructure *s = nullptr; g_object_get(m_object, property, &s, nullptr); return QGstStructure(s); } - bool getBool(const char *property) const { gboolean b = false; g_object_get(m_object, property, &b, nullptr); return b; } - uint getUInt(const char *property) const { guint i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - int getInt(const char *property) const { gint i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - quint64 getUInt64(const char *property) const { guint64 i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - qint64 getInt64(const char *property) const { gint64 i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - float getFloat(const char *property) const { gfloat d = 0; g_object_get(m_object, property, &d, nullptr); return d; } - double getDouble(const char *property) const { gdouble d = 0; g_object_get(m_object, property, &d, nullptr); return d; } - QGstObject getObject(const char *property) const { GstObject *o = nullptr; g_object_get(m_object, property, &o, nullptr); return QGstObject(o, HasRef); } - - void connect(const char *name, GCallback callback, gpointer userData) { g_signal_connect(m_object, name, callback, userData); } - - GstObject *object() const { return m_object; } - const char *name() const { return m_object ? GST_OBJECT_NAME(m_object) : "(null)"; } +private: + QGObjectHandlerConnection connection; }; class QGstElement; @@ -353,48 +463,57 @@ class QGstElement; class QGstPad : public QGstObject { public: - QGstPad() = default; - QGstPad(const QGstObject &o) - : QGstPad(GST_PAD(o.object()), NeedsRef) - {} - QGstPad(GstPad *pad, RefMode mode = NeedsRef) - : QGstObject(&pad->object, mode) - {} - - QGstMutableCaps currentCaps() const - { return QGstMutableCaps(gst_pad_get_current_caps(pad())); } - QGstCaps queryCaps() const - { return QGstCaps(gst_pad_query_caps(pad(), nullptr)); } - - bool isLinked() const { return gst_pad_is_linked(pad()); } - bool link(const QGstPad &sink) const { return gst_pad_link(pad(), sink.pad()) == GST_PAD_LINK_OK; } - bool unlink(const QGstPad &sink) const { return gst_pad_unlink(pad(), sink.pad()); } - bool unlinkPeer() const { return unlink(peer()); } - QGstPad peer() const { return QGstPad(gst_pad_get_peer(pad()), HasRef); } - inline QGstElement parent() const; - - GstPad *pad() const { return GST_PAD_CAST(object()); } - - GstEvent *stickyEvent(GstEventType type) { return gst_pad_get_sticky_event(pad(), type, 0); } - bool sendEvent(GstEvent *event) { return gst_pad_send_event (pad(), event); } + using QGstObject::QGstObject; + QGstPad(const QGstPad &) = default; + QGstPad(QGstPad &&) noexcept = default; + + explicit QGstPad(const QGstObject &o); + explicit QGstPad(GstPad *pad, RefMode mode); + + QGstPad &operator=(const QGstPad &) = default; + QGstPad &operator=(QGstPad &&) noexcept = default; + + QGstCaps currentCaps() const; + QGstCaps queryCaps() const; + + QGstTagListHandle tags() const; + + std::optional<QPlatformMediaPlayer::TrackType> + inferTrackTypeFromName() const; // for decodebin3 etc + + bool isLinked() const; + bool link(const QGstPad &sink) const; + bool unlink(const QGstPad &sink) const; + bool unlinkPeer() const; + QGstPad peer() const; + QGstElement parent() const; + + GstPad *pad() const; + + GstEvent *stickyEvent(GstEventType type); + bool sendEvent(GstEvent *event); template<auto Member, typename T> void addProbe(T *instance, GstPadProbeType type) { - struct Impl { - static GstPadProbeReturn callback(GstPad *pad, GstPadProbeInfo *info, gpointer userData) { - return (static_cast<T *>(userData)->*Member)(QGstPad(pad, NeedsRef), info); - }; + auto callback = [](GstPad *pad, GstPadProbeInfo *info, gpointer userData) { + return (static_cast<T *>(userData)->*Member)(QGstPad(pad, NeedsRef), info); }; - gst_pad_add_probe (pad(), type, Impl::callback, instance, nullptr); + gst_pad_add_probe(pad(), type, callback, instance, nullptr); } - void doInIdleProbe(std::function<void()> work) { + template <typename Functor> + void doInIdleProbe(Functor &&work) + { struct CallbackData { QSemaphore waitDone; - std::function<void()> work; - } cd; - cd.work = work; + Functor work; + }; + + CallbackData cd{ + .waitDone = QSemaphore{}, + .work = std::forward<Functor>(work), + }; auto callback= [](GstPad *, GstPadProbeInfo *, gpointer p) { auto cd = reinterpret_cast<CallbackData*>(p); @@ -409,16 +528,14 @@ public: template<auto Member, typename T> void addEosProbe(T *instance) { - struct Impl { - static GstPadProbeReturn callback(GstPad */*pad*/, GstPadProbeInfo *info, gpointer userData) { - if (GST_EVENT_TYPE(GST_PAD_PROBE_INFO_DATA(info)) != GST_EVENT_EOS) - return GST_PAD_PROBE_PASS; - (static_cast<T *>(userData)->*Member)(); - return GST_PAD_PROBE_REMOVE; - }; + auto callback = [](GstPad *, GstPadProbeInfo *info, gpointer userData) { + if (GST_EVENT_TYPE(GST_PAD_PROBE_INFO_DATA(info)) != GST_EVENT_EOS) + return GST_PAD_PROBE_PASS; + (static_cast<T *>(userData)->*Member)(); + return GST_PAD_PROBE_REMOVE; }; - gst_pad_add_probe (pad(), GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, Impl::callback, instance, nullptr); + gst_pad_add_probe(pad(), GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, callback, instance, nullptr); } }; @@ -426,195 +543,303 @@ class QGstClock : public QGstObject { public: QGstClock() = default; - QGstClock(const QGstObject &o) - : QGstClock(GST_CLOCK(o.object())) - {} - QGstClock(GstClock *clock, RefMode mode = NeedsRef) - : QGstObject(&clock->object, mode) - {} - - GstClock *clock() const { return GST_CLOCK_CAST(object()); } + explicit QGstClock(const QGstObject &o); + explicit QGstClock(GstClock *clock, RefMode mode); - GstClockTime time() const { return gst_clock_get_time(clock()); } + GstClock *clock() const; + GstClockTime time() const; }; +class QGstPipeline; + class QGstElement : public QGstObject { public: - QGstElement() = default; - QGstElement(const QGstObject &o) - : QGstElement(GST_ELEMENT(o.object()), NeedsRef) - {} - QGstElement(GstElement *element, RefMode mode = NeedsRef) - : QGstObject(&element->object, mode) - {} - - QGstElement(const char *factory, const char *name = nullptr) - : QGstElement(gst_element_factory_make(factory, name), NeedsRef) - { - } - - bool linkFiltered(const QGstElement &next, const QGstMutableCaps &caps) - { return gst_element_link_filtered(element(), next.element(), caps.get()); } - bool link(const QGstElement &next) - { return gst_element_link(element(), next.element()); } - bool link(const QGstElement &n1, const QGstElement &n2) - { return gst_element_link_many(element(), n1.element(), n2.element(), nullptr); } - bool link(const QGstElement &n1, const QGstElement &n2, const QGstElement &n3) - { return gst_element_link_many(element(), n1.element(), n2.element(), n3.element(), nullptr); } - bool link(const QGstElement &n1, const QGstElement &n2, const QGstElement &n3, const QGstElement &n4) - { return gst_element_link_many(element(), n1.element(), n2.element(), n3.element(), n4.element(), nullptr); } - bool link(const QGstElement &n1, const QGstElement &n2, const QGstElement &n3, const QGstElement &n4, const QGstElement &n5) - { return gst_element_link_many(element(), n1.element(), n2.element(), n3.element(), n4.element(), n5.element(), nullptr); } - - void unlink(const QGstElement &next) - { gst_element_unlink(element(), next.element()); } - - QGstPad staticPad(const char *name) const { return QGstPad(gst_element_get_static_pad(element(), name), HasRef); } - QGstPad src() const { return staticPad("src"); } - QGstPad sink() const { return staticPad("sink"); } - QGstPad getRequestPad(const char *name) const - { -#if GST_CHECK_VERSION(1,19,1) - return QGstPad(gst_element_request_pad_simple(element(), name), HasRef); -#else - return QGstPad(gst_element_get_request_pad(element(), name), HasRef); -#endif - } - void releaseRequestPad(const QGstPad &pad) const { return gst_element_release_request_pad(element(), pad.pad()); } - - GstState state() const - { - GstState state; - gst_element_get_state(element(), &state, nullptr, 0); - return state; - } - GstStateChangeReturn setState(GstState state) { return gst_element_set_state(element(), state); } - bool setStateSync(GstState state) + using QGstObject::QGstObject; + + QGstElement(const QGstElement &) = default; + QGstElement(QGstElement &&) noexcept = default; + QGstElement &operator=(const QGstElement &) = default; + QGstElement &operator=(QGstElement &&) noexcept = default; + + explicit QGstElement(GstElement *element, RefMode mode); + static QGstElement createFromFactory(const char *factory, const char *name = nullptr); + static QGstElement createFromFactory(GstElementFactory *, const char *name = nullptr); + static QGstElement createFromFactory(const QGstElementFactoryHandle &, + const char *name = nullptr); + static QGstElement createFromDevice(const QGstDeviceHandle &, const char *name = nullptr); + static QGstElement createFromDevice(GstDevice *, const char *name = nullptr); + static QGstElement createFromPipelineDescription(const char *); + static QGstElement createFromPipelineDescription(const QByteArray &); + + static QGstElementFactoryHandle findFactory(const char *); + static QGstElementFactoryHandle findFactory(const QByteArray &name); + + QGstPad staticPad(const char *name) const; + QGstPad src() const; + QGstPad sink() const; + QGstPad getRequestPad(const char *name) const; + void releaseRequestPad(const QGstPad &pad) const; + + GstState state(std::chrono::nanoseconds timeout = std::chrono::seconds(0)) const; + GstStateChangeReturn setState(GstState state); + bool setStateSync(GstState state, std::chrono::nanoseconds timeout = std::chrono::seconds(1)); + bool syncStateWithParent(); + bool finishStateChange(std::chrono::nanoseconds timeout = std::chrono::seconds(5)); + + void lockState(bool locked); + bool isStateLocked() const; + + void sendEvent(GstEvent *event) const; + void sendEos() const; + + std::optional<std::chrono::nanoseconds> duration() const; + std::optional<std::chrono::milliseconds> durationInMs() const; + std::optional<std::chrono::nanoseconds> position() const; + std::optional<std::chrono::milliseconds> positionInMs() const; + std::optional<bool> canSeek() const; + + template <auto Member, typename T> + QGObjectHandlerConnection onPadAdded(T *instance) { - auto change = gst_element_set_state(element(), state); - if (change == GST_STATE_CHANGE_ASYNC) { - change = gst_element_get_state(element(), nullptr, &state, 1000*1e6 /*nano seconds*/); - } -#ifndef QT_NO_DEBUG - if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) - qWarning() << "Could not change state of" << name() << "to" << state << change; -#endif - return change == GST_STATE_CHANGE_SUCCESS; - } - bool syncStateWithParent() { return gst_element_sync_state_with_parent(element()) == TRUE; } - bool finishStateChange() - { - auto change = gst_element_get_state(element(), nullptr, nullptr, 1000*1e6 /*nano seconds*/); -#ifndef QT_NO_DEBUG - if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) - qWarning() << "Could finish change state of" << name(); -#endif - return change == GST_STATE_CHANGE_SUCCESS; - } - - void lockState(bool locked) { gst_element_set_locked_state(element(), locked); } - bool isStateLocked() const { return gst_element_is_locked_state(element()); } - - void sendEvent(GstEvent *event) const { gst_element_send_event(element(), event); } - void sendEos() const { sendEvent(gst_event_new_eos()); } - - template<auto Member, typename T> - void onPadAdded(T *instance) { - struct Impl { - static void callback(GstElement *e, GstPad *pad, gpointer userData) { - (static_cast<T *>(userData)->*Member)(QGstElement(e), QGstPad(pad, NeedsRef)); + struct Impl + { + static void callback(GstElement *e, GstPad *pad, gpointer userData) + { + (static_cast<T *>(userData)->*Member)(QGstElement(e, NeedsRef), + QGstPad(pad, NeedsRef)); }; }; - connect("pad-added", G_CALLBACK(Impl::callback), instance); + return connect("pad-added", G_CALLBACK(Impl::callback), instance); } - template<auto Member, typename T> - void onPadRemoved(T *instance) { - struct Impl { - static void callback(GstElement *e, GstPad *pad, gpointer userData) { - (static_cast<T *>(userData)->*Member)(QGstElement(e), QGstPad(pad, NeedsRef)); + template <auto Member, typename T> + QGObjectHandlerConnection onPadRemoved(T *instance) + { + struct Impl + { + static void callback(GstElement *e, GstPad *pad, gpointer userData) + { + (static_cast<T *>(userData)->*Member)(QGstElement(e, NeedsRef), + QGstPad(pad, NeedsRef)); }; }; - connect("pad-removed", G_CALLBACK(Impl::callback), instance); + return connect("pad-removed", G_CALLBACK(Impl::callback), instance); } - template<auto Member, typename T> - void onNoMorePads(T *instance) { - struct Impl { - static void callback(GstElement *e, gpointer userData) { - (static_cast<T *>(userData)->*Member)(QGstElement(e)); + template <auto Member, typename T> + QGObjectHandlerConnection onNoMorePads(T *instance) + { + struct Impl + { + static void callback(GstElement *e, gpointer userData) + { + (static_cast<T *>(userData)->*Member)(QGstElement(e, NeedsRef)); }; }; - connect("no-more-pads", G_CALLBACK(Impl::callback), instance); + return connect("no-more-pads", G_CALLBACK(Impl::callback), instance); } - GstClockTime baseTime() const { return gst_element_get_base_time(element()); } - void setBaseTime(GstClockTime time) const { gst_element_set_base_time(element(), time); } + GstClockTime baseTime() const; + void setBaseTime(GstClockTime time) const; + + GstElement *element() const; + + QGstElement getParent() const; + QGstPipeline getPipeline() const; + void dumpPipelineGraph(const char *filename) const; - GstElement *element() const { return GST_ELEMENT_CAST(m_object); } +private: + QGstQueryHandle &positionQuery() const; + mutable QGstQueryHandle m_positionQuery; }; -inline QGstElement QGstPad::parent() const +template <typename... Ts> +std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> +qLinkGstElements(const Ts &...ts) { - return QGstElement(gst_pad_get_parent_element(pad()), HasRef); + bool link_success = [&] { + if constexpr (sizeof...(Ts) == 2) + return gst_element_link(ts.element()...); + else + return gst_element_link_many(ts.element()..., nullptr); + }(); + + if (Q_UNLIKELY(!link_success)) { + qWarning() << "qLinkGstElements: could not link elements: " + << std::initializer_list<const char *>{ + (GST_ELEMENT_NAME(ts.element()))..., + }; + } +} + +template <typename... Ts> +std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> +qUnlinkGstElements(const Ts &...ts) +{ + if constexpr (sizeof...(Ts) == 2) + gst_element_unlink(ts.element()...); + else + gst_element_unlink_many(ts.element()..., nullptr); } class QGstBin : public QGstElement { public: - QGstBin() = default; - QGstBin(const QGstObject &o) - : QGstBin(GST_BIN(o.object()), NeedsRef) - {} - QGstBin(const char *name) - : QGstElement(gst_bin_new(name), NeedsRef) + using QGstElement::QGstElement; + QGstBin(const QGstBin &) = default; + QGstBin(QGstBin &&) noexcept = default; + QGstBin &operator=(const QGstBin &) = default; + QGstBin &operator=(QGstBin &&) noexcept = default; + + explicit QGstBin(GstBin *bin, RefMode mode = NeedsRef); + static QGstBin create(const char *name); + static QGstBin createFromFactory(const char *factory, const char *name); + static QGstBin createFromPipelineDescription(const QByteArray &pipelineDescription, + const char *name = nullptr, + bool ghostUnlinkedPads = false); + static QGstBin createFromPipelineDescription(const char *pipelineDescription, + const char *name = nullptr, + bool ghostUnlinkedPads = false); + + template <typename... Ts> + std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> add(const Ts &...ts) { + if constexpr (sizeof...(Ts) == 1) + gst_bin_add(bin(), ts.element()...); + else + gst_bin_add_many(bin(), ts.element()..., nullptr); } - QGstBin(GstBin *bin, RefMode mode = NeedsRef) - : QGstElement(&bin->element, mode) - {} - - void add(const QGstElement &element) - { gst_bin_add(bin(), element.element()); } - void add(const QGstElement &e1, const QGstElement &e2) - { gst_bin_add_many(bin(), e1.element(), e2.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3, const QGstElement &e4) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), e4.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3, const QGstElement &e4, const QGstElement &e5) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), e4.element(), e5.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3, const QGstElement &e4, const QGstElement &e5, const QGstElement &e6) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), e4.element(), e5.element(), e6.element(), nullptr); } - void remove(const QGstElement &element) - { gst_bin_remove(bin(), element.element()); } - - GstBin *bin() const { return GST_BIN_CAST(m_object); } - - void addGhostPad(const QGstElement &child, const char *name) + + template <typename... Ts> + std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> remove(const Ts &...ts) { - addGhostPad(name, child.staticPad(name)); + if constexpr (sizeof...(Ts) == 1) + gst_bin_remove(bin(), ts.element()...); + else + gst_bin_remove_many(bin(), ts.element()..., nullptr); } - void addGhostPad(const char *name, const QGstPad &pad) + + template <typename... Ts> + std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> + stopAndRemoveElements(Ts... ts) { - gst_element_add_pad(element(), gst_ghost_pad_new(name, pad.pad())); + bool stateChangeSuccessful = (ts.setStateSync(GST_STATE_NULL) && ...); + Q_ASSERT(stateChangeSuccessful); + remove(ts...); } + + GstBin *bin() const; + + void addGhostPad(const QGstElement &child, const char *name); + void addGhostPad(const char *name, const QGstPad &pad); + + bool syncChildrenState(); + + void dumpGraph(const char *fileNamePrefix) const; + + QGstElement findByName(const char *); +}; + +class QGstBaseSink : public QGstElement +{ +public: + using QGstElement::QGstElement; + + explicit QGstBaseSink(GstBaseSink *, RefMode); + + QGstBaseSink(const QGstBaseSink &) = default; + QGstBaseSink(QGstBaseSink &&) noexcept = default; + QGstBaseSink &operator=(const QGstBaseSink &) = default; + QGstBaseSink &operator=(QGstBaseSink &&) noexcept = default; + + void setSync(bool); + + GstBaseSink *baseSink() const; +}; + +class QGstBaseSrc : public QGstElement +{ +public: + using QGstElement::QGstElement; + + explicit QGstBaseSrc(GstBaseSrc *, RefMode); + + QGstBaseSrc(const QGstBaseSrc &) = default; + QGstBaseSrc(QGstBaseSrc &&) noexcept = default; + QGstBaseSrc &operator=(const QGstBaseSrc &) = default; + QGstBaseSrc &operator=(QGstBaseSrc &&) noexcept = default; + + GstBaseSrc *baseSrc() const; +}; + +class QGstAppSink : public QGstBaseSink +{ +public: + using QGstBaseSink::QGstBaseSink; + + explicit QGstAppSink(GstAppSink *, RefMode); + + QGstAppSink(const QGstAppSink &) = default; + QGstAppSink(QGstAppSink &&) noexcept = default; + QGstAppSink &operator=(const QGstAppSink &) = default; + QGstAppSink &operator=(QGstAppSink &&) noexcept = default; + + static QGstAppSink create(const char *name); + + GstAppSink *appSink() const; + + void setMaxBuffers(int); +# if GST_CHECK_VERSION(1, 24, 0) + void setMaxBufferTime(std::chrono::nanoseconds); +# endif + + void setCaps(const QGstCaps &caps); + void setCallbacks(GstAppSinkCallbacks &callbacks, gpointer user_data, GDestroyNotify notify); + + QGstSampleHandle pullSample(); }; -inline QGstStructure QGValue::toStructure() const +class QGstAppSrc : public QGstBaseSrc { - if (!value || !GST_VALUE_HOLDS_STRUCTURE(value)) - return QGstStructure(); - return QGstStructure(gst_value_get_structure(value)); +public: + using QGstBaseSrc::QGstBaseSrc; + + explicit QGstAppSrc(GstAppSrc *, RefMode); + + QGstAppSrc(const QGstAppSrc &) = default; + QGstAppSrc(QGstAppSrc &&) noexcept = default; + QGstAppSrc &operator=(const QGstAppSrc &) = default; + QGstAppSrc &operator=(QGstAppSrc &&) noexcept = default; + + static QGstAppSrc create(const char *name); + + GstAppSrc *appSrc() const; + + void setCallbacks(GstAppSrcCallbacks &callbacks, gpointer user_data, GDestroyNotify notify); + + GstFlowReturn pushBuffer(GstBuffer *); // take ownership +}; + +inline GstClockTime qGstClockTimeFromChrono(std::chrono::nanoseconds ns) +{ + return ns.count(); } -inline QGstCaps QGValue::toCaps() const +QString qGstErrorMessageCannotFindElement(std::string_view element); + +template <typename Arg, typename... Args> +std::optional<QString> qGstErrorMessageIfElementsNotAvailable(const Arg &arg, Args... args) { - if (!value || !GST_VALUE_HOLDS_CAPS(value)) - return QGstCaps(); - return QGstCaps(gst_value_get_caps(value)); + QGstElementFactoryHandle factory = QGstElement::findFactory(arg); + if (!factory) + return qGstErrorMessageCannotFindElement(arg); + + if constexpr (sizeof...(args) != 0) + return qGstErrorMessageIfElementsNotAvailable(args...); + else + return std::nullopt; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp b/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp new file mode 100644 index 000000000..5779ba8b1 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp @@ -0,0 +1,240 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qgstappsource_p.h" + +#include <QtCore/qdebug.h> +#include <QtCore/qloggingcategory.h> + +#include <common/qgstutils_p.h> + +static Q_LOGGING_CATEGORY(qLcAppSrc, "qt.multimedia.appsrc") + +QT_BEGIN_NAMESPACE + +QMaybe<QGstAppSource *> QGstAppSource::create(QObject *parent) +{ + QGstAppSrc appsrc = QGstAppSrc::create("appsrc"); + if (!appsrc) + return qGstErrorMessageCannotFindElement("appsrc"); + + return new QGstAppSource(appsrc, parent); +} + +QGstAppSource::QGstAppSource(QGstAppSrc appsrc, QObject *parent) + : QObject(parent), m_appSrc(std::move(appsrc)) +{ + m_appSrc.set("emit-signals", false); +} + +QGstAppSource::~QGstAppSource() +{ + m_appSrc.setStateSync(GST_STATE_NULL); + streamDestroyed(); + qCDebug(qLcAppSrc) << "~QGstAppSrc"; +} + +bool QGstAppSource::setup(QIODevice *stream, qint64 offset) +{ + QMutexLocker locker(&m_mutex); + + if (m_appSrc.isNull()) + return false; + + if (!setStream(stream, offset)) + return false; + + GstAppSrcCallbacks callbacks{}; + callbacks.need_data = QGstAppSource::on_need_data; + callbacks.enough_data = QGstAppSource::on_enough_data; + callbacks.seek_data = QGstAppSource::on_seek_data; + + m_appSrc.setCallbacks(callbacks, this, nullptr); + + GstAppSrc *appSrc = m_appSrc.appSrc(); + m_maxBytes = gst_app_src_get_max_bytes(appSrc); + + if (m_sequential) + m_streamType = GST_APP_STREAM_TYPE_STREAM; + else + m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; + gst_app_src_set_stream_type(appSrc, m_streamType); + gst_app_src_set_size(appSrc, m_sequential ? -1 : m_stream->size() - m_offset); + + return true; +} + +void QGstAppSource::setExternalAppSrc(QGstAppSrc appsrc) +{ + QMutexLocker locker(&m_mutex); + m_appSrc = std::move(appsrc); +} + +bool QGstAppSource::setStream(QIODevice *stream, qint64 offset) +{ + if (m_stream) { + disconnect(m_stream, &QIODevice::readyRead, this, &QGstAppSource::onDataReady); + disconnect(m_stream, &QIODevice::destroyed, this, &QGstAppSource::streamDestroyed); + m_stream = nullptr; + } + + m_dataRequestSize = 0; + m_sequential = true; + m_maxBytes = 0; + + if (stream) { + if (!stream->isOpen() && !stream->open(QIODevice::ReadOnly)) + return false; + m_stream = stream; + connect(m_stream, &QIODevice::destroyed, this, &QGstAppSource::streamDestroyed); + connect(m_stream, &QIODevice::readyRead, this, &QGstAppSource::onDataReady); + m_sequential = m_stream->isSequential(); + m_offset = offset; + } + return true; +} + +bool QGstAppSource::isStreamValid() const +{ + return m_stream != nullptr && m_stream->isOpen(); +} + +QGstElement QGstAppSource::element() const +{ + return m_appSrc; +} + +void QGstAppSource::onDataReady() +{ + qCDebug(qLcAppSrc) << "onDataReady" << m_stream->bytesAvailable() << m_stream->size(); + pushData(); +} + +void QGstAppSource::streamDestroyed() +{ + qCDebug(qLcAppSrc) << "stream destroyed"; + m_stream = nullptr; + m_dataRequestSize = 0; + sendEOS(); +} + +void QGstAppSource::pushData() +{ + if (m_appSrc.isNull() || !m_dataRequestSize) { + qCDebug(qLcAppSrc) << "push data: return immediately" << m_appSrc.isNull() + << m_dataRequestSize; + return; + } + + Q_ASSERT(m_stream); + + qCDebug(qLcAppSrc) << "pushData" << m_stream; + if ((m_stream && m_stream->atEnd())) { + sendEOS(); + qCDebug(qLcAppSrc) << "end pushData" << m_stream; + return; + } + + qint64 size = m_stream->bytesAvailable(); + + if (!m_dataRequestSize) + m_dataRequestSize = m_maxBytes; + size = qMin(size, (qint64)m_dataRequestSize); + qCDebug(qLcAppSrc) << " reading" << size << "bytes" << size << m_dataRequestSize; + + GstBuffer* buffer = gst_buffer_new_and_alloc(size); + + if (m_sequential) + buffer->offset = bytesReadSoFar; + else + buffer->offset = m_stream->pos(); + + GstMapInfo mapInfo; + gst_buffer_map(buffer, &mapInfo, GST_MAP_WRITE); + void* bufferData = mapInfo.data; + + qint64 bytesRead; + bytesRead = m_stream->read((char *)bufferData, size); + + buffer->offset_end = buffer->offset + bytesRead - 1; + bytesReadSoFar += bytesRead; + + gst_buffer_unmap(buffer, &mapInfo); + qCDebug(qLcAppSrc) << "pushing bytes into gstreamer" << buffer->offset << bytesRead; + if (bytesRead == 0) { + gst_buffer_unref(buffer); + sendEOS(); + qCDebug(qLcAppSrc) << "end pushData" << m_stream; + return; + } + + GstFlowReturn ret = m_appSrc.pushBuffer(buffer); + switch (ret) { + case GST_FLOW_OK: + break; + + default: + qWarning() << "QGstAppSrc: push buffer error -" << gst_flow_get_name(ret); + break; + } + + qCDebug(qLcAppSrc) << "end pushData" << m_stream; +} + +bool QGstAppSource::doSeek(qint64 value) +{ + if (isStreamValid()) + return m_stream->seek(value + m_offset); + return false; +} + +gboolean QGstAppSource::on_seek_data(GstAppSrc *, guint64 arg0, gpointer userdata) +{ + // we do get some spurious seeks to INT_MAX, ignore those + if (arg0 == std::numeric_limits<quint64>::max()) + return true; + + QGstAppSource *self = reinterpret_cast<QGstAppSource *>(userdata); + Q_ASSERT(self); + + QMutexLocker locker(&self->m_mutex); + + if (self->m_sequential) + return false; + + self->doSeek(arg0); + return true; +} + +void QGstAppSource::on_enough_data(GstAppSrc *, gpointer userdata) +{ + qCDebug(qLcAppSrc) << "on_enough_data"; + QGstAppSource *self = static_cast<QGstAppSource *>(userdata); + Q_ASSERT(self); + QMutexLocker locker(&self->m_mutex); + self->m_dataRequestSize = 0; +} + +void QGstAppSource::on_need_data(GstAppSrc *, guint arg0, gpointer userdata) +{ + qCDebug(qLcAppSrc) << "on_need_data requesting bytes" << arg0; + QGstAppSource *self = static_cast<QGstAppSource *>(userdata); + Q_ASSERT(self); + QMutexLocker locker(&self->m_mutex); + self->m_dataRequestSize = arg0; + self->pushData(); + qCDebug(qLcAppSrc) << "done on_need_data"; +} + +void QGstAppSource::sendEOS() +{ + qCDebug(qLcAppSrc) << "sending EOS"; + if (m_appSrc.isNull()) + return; + + gst_app_src_end_of_stream(GST_APP_SRC(m_appSrc.element())); +} + +QT_END_NAMESPACE + +#include "moc_qgstappsource_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsource_p.h b/src/plugins/multimedia/gstreamer/common/qgstappsource_p.h new file mode 100644 index 000000000..b181212d2 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgstappsource_p.h @@ -0,0 +1,77 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGSTAPPSRC_H +#define QGSTAPPSRC_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + + +#include <QtCore/qobject.h> +#include <QtCore/qiodevice.h> +#include <QtCore/qatomic.h> +#include <QtCore/qmutex.h> + +#include <QtMultimedia/private/qtmultimediaglobal_p.h> + +#include <common/qgst_p.h> +#include <gst/app/gstappsrc.h> + +QT_BEGIN_NAMESPACE + +class QGstAppSource : public QObject +{ + Q_OBJECT +public: + static QMaybe<QGstAppSource *> create(QObject *parent = nullptr); + ~QGstAppSource(); + + bool setup(QIODevice *stream = nullptr, qint64 offset = 0); + + void setExternalAppSrc(QGstAppSrc); + QGstElement element() const; + +private Q_SLOTS: + void onDataReady(); + void streamDestroyed(); + +private: + bool doSeek(qint64); + void pushData(); + + QGstAppSource(QGstAppSrc appsrc, QObject *parent); + + bool setStream(QIODevice *, qint64 offset); + bool isStreamValid() const; + + static gboolean on_seek_data(GstAppSrc *element, guint64 arg0, gpointer userdata); + static void on_enough_data(GstAppSrc *element, gpointer userdata); + static void on_need_data(GstAppSrc *element, uint arg0, gpointer userdata); + + void sendEOS(); + + mutable QMutex m_mutex; + + QIODevice *m_stream = nullptr; + + QGstAppSrc m_appSrc; + bool m_sequential = true; + GstAppStreamType m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; + qint64 m_offset = 0; + qint64 m_maxBytes = 0; + qint64 bytesReadSoFar = 0; + QAtomicInteger<unsigned int> m_dataRequestSize = 0; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsrc.cpp b/src/plugins/multimedia/gstreamer/common/qgstappsrc.cpp deleted file mode 100644 index 992e3d5a9..000000000 --- a/src/plugins/multimedia/gstreamer/common/qgstappsrc.cpp +++ /dev/null @@ -1,308 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QDebug> - -#include "qgstappsrc_p.h" -#include "qgstutils_p.h" -#include "qnetworkreply.h" -#include "qloggingcategory.h" - -Q_LOGGING_CATEGORY(qLcAppSrc, "qt.multimedia.appsrc") - -QGstAppSrc::QGstAppSrc(QObject *parent) - : QObject(parent) -{ - m_appSrc = QGstElement("appsrc", "appsrc"); - if (m_appSrc.isNull()) - qWarning() << "Could not create GstAppSrc."; -} - -QGstAppSrc::~QGstAppSrc() -{ - m_appSrc.setStateSync(GST_STATE_NULL); - streamDestroyed(); - qCDebug(qLcAppSrc) << "~QGstAppSrc"; -} - -bool QGstAppSrc::setup(QIODevice *stream, qint64 offset) -{ - if (m_appSrc.isNull()) - return false; - - if (!setStream(stream, offset)) - return false; - - auto *appSrc = GST_APP_SRC(m_appSrc.element()); - GstAppSrcCallbacks m_callbacks; - memset(&m_callbacks, 0, sizeof(GstAppSrcCallbacks)); - m_callbacks.need_data = &QGstAppSrc::on_need_data; - m_callbacks.enough_data = &QGstAppSrc::on_enough_data; - m_callbacks.seek_data = &QGstAppSrc::on_seek_data; - gst_app_src_set_callbacks(appSrc, (GstAppSrcCallbacks*)&m_callbacks, this, nullptr); - - m_maxBytes = gst_app_src_get_max_bytes(appSrc); - m_suspended = false; - - if (m_sequential) - m_streamType = GST_APP_STREAM_TYPE_STREAM; - else - m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; - gst_app_src_set_stream_type(appSrc, m_streamType); - gst_app_src_set_size(appSrc, m_sequential ? -1 : m_stream->size() - m_offset); - - m_networkReply = qobject_cast<QNetworkReply *>(m_stream); - m_noMoreData = true; - - return true; -} - -void QGstAppSrc::setAudioFormat(const QAudioFormat &f) -{ - m_format = f; - if (!m_format.isValid()) - return; - - auto caps = QGstUtils::capsForAudioFormat(m_format); - Q_ASSERT(!caps.isNull()); - m_appSrc.set("caps", caps); - m_appSrc.set("format", GST_FORMAT_TIME); -} - -void QGstAppSrc::setExternalAppSrc(const QGstElement &appsrc) -{ - m_appSrc = appsrc; -} - -bool QGstAppSrc::setStream(QIODevice *stream, qint64 offset) -{ - if (m_stream) { - disconnect(m_stream, SIGNAL(readyRead()), this, SLOT(onDataReady())); - disconnect(m_stream, SIGNAL(destroyed()), this, SLOT(streamDestroyed())); - m_stream = nullptr; - } - - m_dataRequestSize = 0; - m_sequential = true; - m_maxBytes = 0; - streamedSamples = 0; - - if (stream) { - if (!stream->isOpen() && !stream->open(QIODevice::ReadOnly)) - return false; - m_stream = stream; - connect(m_stream, SIGNAL(destroyed()), SLOT(streamDestroyed())); - connect(m_stream, SIGNAL(readyRead()), this, SLOT(onDataReady())); - m_sequential = m_stream->isSequential(); - m_offset = offset; - } - return true; -} - -QGstElement QGstAppSrc::element() -{ - return m_appSrc; -} - -void QGstAppSrc::write(const char *data, qsizetype size) -{ - qCDebug(qLcAppSrc) << "write" << size << m_noMoreData << m_dataRequestSize; - if (!size) - return; - Q_ASSERT(!m_stream); - m_buffer.append(data, size); - m_noMoreData = false; - pushData(); -} - -void QGstAppSrc::onDataReady() -{ - qCDebug(qLcAppSrc) << "onDataReady" << m_stream->bytesAvailable() << m_stream->size(); - pushData(); -} - -void QGstAppSrc::streamDestroyed() -{ - qCDebug(qLcAppSrc) << "stream destroyed"; - m_stream = nullptr; - m_dataRequestSize = 0; - streamedSamples = 0; - sendEOS(); -} - -void QGstAppSrc::pushData() -{ - if (m_appSrc.isNull() || !m_dataRequestSize || m_suspended) { - qCDebug(qLcAppSrc) << "push data: return immediately" << m_appSrc.isNull() << m_dataRequestSize << m_suspended; - return; - } - - qCDebug(qLcAppSrc) << "pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - if ((m_stream && m_stream->atEnd())) { - eosOrIdle(); - qCDebug(qLcAppSrc) << "end pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - return; - } - - qint64 size; - if (m_stream) - size = m_stream->bytesAvailable(); - else - size = m_buffer.size(); - - if (!m_dataRequestSize) - m_dataRequestSize = m_maxBytes; - size = qMin(size, (qint64)m_dataRequestSize); - qCDebug(qLcAppSrc) << " reading" << size << "bytes" << size << m_dataRequestSize; - - GstBuffer* buffer = gst_buffer_new_and_alloc(size); - - if (m_sequential || !m_stream) - buffer->offset = bytesReadSoFar; - else - buffer->offset = m_stream->pos(); - - if (m_format.isValid()) { - // timestamp raw audio data - uint nSamples = size/m_format.bytesPerFrame(); - - GST_BUFFER_TIMESTAMP(buffer) = gst_util_uint64_scale(streamedSamples, GST_SECOND, m_format.sampleRate()); - GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(nSamples, GST_SECOND, m_format.sampleRate()); - streamedSamples += nSamples; - } - - GstMapInfo mapInfo; - gst_buffer_map(buffer, &mapInfo, GST_MAP_WRITE); - void* bufferData = mapInfo.data; - - qint64 bytesRead; - if (m_stream) - bytesRead = m_stream->read((char*)bufferData, size); - else - bytesRead = m_buffer.read((char*)bufferData, size); - buffer->offset_end = buffer->offset + bytesRead - 1; - bytesReadSoFar += bytesRead; - - gst_buffer_unmap(buffer, &mapInfo); - qCDebug(qLcAppSrc) << "pushing bytes into gstreamer" << buffer->offset << bytesRead; - if (bytesRead == 0) { - gst_buffer_unref(buffer); - eosOrIdle(); - qCDebug(qLcAppSrc) << "end pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - return; - } - m_noMoreData = false; - emit bytesProcessed(bytesRead); - - GstFlowReturn ret = gst_app_src_push_buffer(GST_APP_SRC(m_appSrc.element()), buffer); - if (ret == GST_FLOW_ERROR) { - qWarning() << "QGstAppSrc: push buffer error"; - } else if (ret == GST_FLOW_FLUSHING) { - qWarning() << "QGstAppSrc: push buffer wrong state"; - } - qCDebug(qLcAppSrc) << "end pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - -} - -bool QGstAppSrc::doSeek(qint64 value) -{ - if (isStreamValid()) - return m_stream->seek(value + m_offset); - return false; -} - - -gboolean QGstAppSrc::on_seek_data(GstAppSrc *, guint64 arg0, gpointer userdata) -{ - // we do get some spurious seeks to INT_MAX, ignore those - if (arg0 == std::numeric_limits<quint64>::max()) - return true; - - QGstAppSrc *self = reinterpret_cast<QGstAppSrc*>(userdata); - Q_ASSERT(self); - if (self->m_sequential) - return false; - - QMetaObject::invokeMethod(self, "doSeek", Qt::AutoConnection, Q_ARG(qint64, arg0)); - return true; -} - -void QGstAppSrc::on_enough_data(GstAppSrc *, gpointer userdata) -{ - qCDebug(qLcAppSrc) << "on_enough_data"; - QGstAppSrc *self = static_cast<QGstAppSrc*>(userdata); - Q_ASSERT(self); - self->m_dataRequestSize = 0; -} - -void QGstAppSrc::on_need_data(GstAppSrc *, guint arg0, gpointer userdata) -{ - qCDebug(qLcAppSrc) << "on_need_data requesting bytes" << arg0; - QGstAppSrc *self = static_cast<QGstAppSrc*>(userdata); - Q_ASSERT(self); - self->m_dataRequestSize = arg0; - QMetaObject::invokeMethod(self, "pushData", Qt::AutoConnection); - qCDebug(qLcAppSrc) << "done on_need_data"; -} - -void QGstAppSrc::sendEOS() -{ - qCDebug(qLcAppSrc) << "sending EOS"; - if (m_appSrc.isNull()) - return; - - gst_app_src_end_of_stream(GST_APP_SRC(m_appSrc.element())); -} - -void QGstAppSrc::eosOrIdle() -{ - qCDebug(qLcAppSrc) << "eosOrIdle"; - if (m_appSrc.isNull()) - return; - - if (!m_sequential) { - sendEOS(); - return; - } - if (m_noMoreData) - return; - qCDebug(qLcAppSrc) << " idle!"; - m_noMoreData = true; - emit noMoreData(); -} diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsrc_p.h b/src/plugins/multimedia/gstreamer/common/qgstappsrc_p.h deleted file mode 100644 index 373dbea65..000000000 --- a/src/plugins/multimedia/gstreamer/common/qgstappsrc_p.h +++ /dev/null @@ -1,132 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QGSTAPPSRC_H -#define QGSTAPPSRC_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qtmultimediaglobal_p.h> -#include <qaudioformat.h> - -#include <QtCore/qobject.h> -#include <QtCore/qiodevice.h> -#include <QtCore/private/qringbuffer_p.h> -#include <QtCore/qatomic.h> - -#include <qgst_p.h> -#include <gst/app/gstappsrc.h> - -QT_BEGIN_NAMESPACE - -class QNetworkReply; - -class Q_MULTIMEDIA_EXPORT QGstAppSrc : public QObject -{ - Q_OBJECT -public: - QGstAppSrc(QObject *parent = 0); - ~QGstAppSrc(); - - bool setup(QIODevice *stream = nullptr, qint64 offset = 0); - void setAudioFormat(const QAudioFormat &f); - - void setExternalAppSrc(const QGstElement &appsrc); - QGstElement element(); - - void write(const char *data, qsizetype size); - - bool canAcceptMoreData() { return m_noMoreData || m_dataRequestSize != 0; } - - void suspend() { m_suspended = true; } - void resume() { m_suspended = false; m_noMoreData = true; } - -Q_SIGNALS: - void bytesProcessed(int bytes); - void noMoreData(); - -private Q_SLOTS: - void pushData(); - bool doSeek(qint64); - void onDataReady(); - - void streamDestroyed(); -private: - bool setStream(QIODevice *, qint64 offset); - bool isStreamValid() const - { - return m_stream != nullptr && m_stream->isOpen(); - } - - static gboolean on_seek_data(GstAppSrc *element, guint64 arg0, gpointer userdata); - static void on_enough_data(GstAppSrc *element, gpointer userdata); - static void on_need_data(GstAppSrc *element, uint arg0, gpointer userdata); - - void sendEOS(); - void eosOrIdle(); - - QIODevice *m_stream = nullptr; - QNetworkReply *m_networkReply = nullptr; - QRingBuffer m_buffer; - QAudioFormat m_format; - - QGstElement m_appSrc; - bool m_sequential = true; - bool m_suspended = false; - bool m_noMoreData = false; - GstAppStreamType m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; - qint64 m_offset = 0; - qint64 m_maxBytes = 0; - qint64 bytesReadSoFar = 0; - QAtomicInteger<unsigned int> m_dataRequestSize = 0; - int streamedSamples = 0; -}; - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp b/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp index 1f20fe4ec..a2f8e91c2 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp @@ -1,94 +1,73 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -#include <QtCore/qmap.h> -#include <QtCore/qtimer.h> -#include <QtCore/qmutex.h> -#include <QtCore/qlist.h> #include <QtCore/qabstracteventdispatcher.h> #include <QtCore/qcoreapplication.h> +#include <QtCore/qlist.h> +#include <QtCore/qloggingcategory.h> +#include <QtCore/qmap.h> +#include <QtCore/qmutex.h> #include <QtCore/qproperty.h> +#include <QtCore/qtimer.h> #include "qgstpipeline_p.h" #include "qgstreamermessage_p.h" QT_BEGIN_NAMESPACE -class QGstPipelinePrivate : public QObject +static Q_LOGGING_CATEGORY(qLcGstPipeline, "qt.multimedia.gstpipeline"); + +static constexpr GstSeekFlags rateChangeSeekFlags = +#if GST_CHECK_VERSION(1, 18, 0) + GST_SEEK_FLAG_INSTANT_RATE_CHANGE; +#else + GST_SEEK_FLAG_FLUSH; +#endif + +class QGstPipelinePrivate { - Q_OBJECT public: - - int m_ref = 0; - guint m_tag = 0; + guint m_eventSourceID = 0; GstBus *m_bus = nullptr; - QTimer *m_intervalTimer = nullptr; + std::unique_ptr<QTimer> m_intervalTimer; QMutex filterMutex; QList<QGstreamerSyncMessageFilter*> syncFilters; QList<QGstreamerBusMessageFilter*> busFilters; - bool inStoppedState = true; - mutable qint64 m_position = 0; + mutable std::chrono::nanoseconds m_position{}; double m_rate = 1.; - bool m_flushOnConfigChanges = false; - bool m_pendingFlush = false; int m_configCounter = 0; GstState m_savedState = GST_STATE_NULL; - QGstPipelinePrivate(GstBus* bus, QObject* parent = 0); + explicit QGstPipelinePrivate(GstBus *bus); ~QGstPipelinePrivate(); - void ref() { ++ m_ref; } - void deref() { if (!--m_ref) delete this; } - void installMessageFilter(QGstreamerSyncMessageFilter *filter); void removeMessageFilter(QGstreamerSyncMessageFilter *filter); void installMessageFilter(QGstreamerBusMessageFilter *filter); void removeMessageFilter(QGstreamerBusMessageFilter *filter); - static GstBusSyncReply syncGstBusFilter(GstBus* bus, GstMessage* message, QGstPipelinePrivate *d) + void processMessage(const QGstreamerMessage &msg) { + for (QGstreamerBusMessageFilter *filter : std::as_const(busFilters)) { + if (filter->processBusMessage(msg)) + break; + } + } + +private: + static GstBusSyncReply syncGstBusFilter(GstBus *bus, GstMessage *message, + QGstPipelinePrivate *d) + { + if (!message) + return GST_BUS_PASS; + Q_UNUSED(bus); QMutexLocker lock(&d->filterMutex); - for (QGstreamerSyncMessageFilter *filter : qAsConst(d->syncFilters)) { - if (filter->processSyncMessage(QGstreamerMessage(message))) { + for (QGstreamerSyncMessageFilter *filter : std::as_const(d->syncFilters)) { + if (filter->processSyncMessage( + QGstreamerMessage{ message, QGstreamerMessage::NeedsRef })) { gst_message_unref(message); return GST_BUS_DROP; } @@ -97,59 +76,45 @@ public: return GST_BUS_PASS; } -private Q_SLOTS: - void interval() - { - GstMessage* message; - while ((message = gst_bus_poll(m_bus, GST_MESSAGE_ANY, 0)) != nullptr) { - processMessage(message); - gst_message_unref(message); - } - } - void doProcessMessage(const QGstreamerMessage& msg) + void processMessage(GstMessage *message) { - for (QGstreamerBusMessageFilter *filter : qAsConst(busFilters)) { - if (filter->processBusMessage(msg)) - break; - } - } + if (!message) + return; -private: - void processMessage(GstMessage* message) - { - QGstreamerMessage msg(message); - doProcessMessage(msg); - } + QGstreamerMessage msg{ + message, + QGstreamerMessage::NeedsRef, + }; - void queueMessage(GstMessage* message) - { - QGstreamerMessage msg(message); - QMetaObject::invokeMethod(this, "doProcessMessage", Qt::QueuedConnection, - Q_ARG(QGstreamerMessage, msg)); + processMessage(msg); } - static gboolean busCallback(GstBus *bus, GstMessage *message, gpointer data) + static gboolean busCallback(GstBus *, GstMessage *message, gpointer data) { - Q_UNUSED(bus); - static_cast<QGstPipelinePrivate *>(data)->queueMessage(message); + static_cast<QGstPipelinePrivate *>(data)->processMessage(message); return TRUE; } }; -QGstPipelinePrivate::QGstPipelinePrivate(GstBus* bus, QObject* parent) - : QObject(parent), - m_bus(bus) +QGstPipelinePrivate::QGstPipelinePrivate(GstBus *bus) : m_bus(bus) { // glib event loop can be disabled either by env variable or QT_NO_GLIB define, so check the dispacher QAbstractEventDispatcher *dispatcher = QCoreApplication::eventDispatcher(); const bool hasGlib = dispatcher && dispatcher->inherits("QEventDispatcherGlib"); if (!hasGlib) { - m_intervalTimer = new QTimer(this); + m_intervalTimer = std::make_unique<QTimer>(); m_intervalTimer->setInterval(250); - connect(m_intervalTimer, SIGNAL(timeout()), SLOT(interval())); + QObject::connect(m_intervalTimer.get(), &QTimer::timeout, m_intervalTimer.get(), [this] { + GstMessage *message; + while ((message = gst_bus_poll(m_bus, GST_MESSAGE_ANY, 0)) != nullptr) { + processMessage(message); + gst_message_unref(message); + } + }); m_intervalTimer->start(); } else { - m_tag = gst_bus_add_watch_full(bus, G_PRIORITY_DEFAULT, busCallback, this, nullptr); + m_eventSourceID = + gst_bus_add_watch_full(bus, G_PRIORITY_DEFAULT, busCallback, this, nullptr); } gst_bus_set_sync_handler(bus, (GstBusSyncHandler)syncGstBusFilter, this, nullptr); @@ -157,9 +122,9 @@ QGstPipelinePrivate::QGstPipelinePrivate(GstBus* bus, QObject* parent) QGstPipelinePrivate::~QGstPipelinePrivate() { - delete m_intervalTimer; + m_intervalTimer.reset(); - if (m_tag) + if (m_eventSourceID) gst_bus_remove_watch(m_bus); gst_bus_set_sync_handler(m_bus, nullptr, nullptr, nullptr); @@ -195,125 +160,121 @@ void QGstPipelinePrivate::removeMessageFilter(QGstreamerBusMessageFilter *filter busFilters.removeAll(filter); } -QGstPipeline::QGstPipeline(const QGstPipeline &o) - : QGstBin(o.bin(), NeedsRef), - d(o.d) -{ - if (d) - d->ref(); -} - -QGstPipeline &QGstPipeline::operator=(const QGstPipeline &o) -{ - if (this == &o) - return *this; - if (o.d) - o.d->ref(); - if (d) - d->deref(); - QGstBin::operator=(o); - d = o.d; - return *this; -} - -QGstPipeline::QGstPipeline(const char *name) - : QGstBin(GST_BIN(gst_pipeline_new(name)), NeedsRef) -{ - d = new QGstPipelinePrivate(gst_pipeline_get_bus(pipeline())); - d->ref(); -} - -QGstPipeline::QGstPipeline(GstPipeline *p) - : QGstBin(&p->bin, NeedsRef) -{ - d = new QGstPipelinePrivate(gst_pipeline_get_bus(pipeline())); - d->ref(); -} - -QGstPipeline::~QGstPipeline() +QGstPipeline QGstPipeline::create(const char *name) { - if (d) - d->deref(); + GstPipeline *pipeline = qGstCheckedCast<GstPipeline>(gst_pipeline_new(name)); + return adopt(pipeline); } -bool QGstPipeline::inStoppedState() const +QGstPipeline QGstPipeline::adopt(GstPipeline *pipeline) { - Q_ASSERT(d); - return d->inStoppedState; + QGstPipelinePrivate *d = new QGstPipelinePrivate(gst_pipeline_get_bus(pipeline)); + g_object_set_data_full(qGstCheckedCast<GObject>(pipeline), "pipeline-private", d, + [](gpointer ptr) { + delete reinterpret_cast<QGstPipelinePrivate *>(ptr); + return; + }); + + return QGstPipeline{ + pipeline, + QGstPipeline::NeedsRef, + }; } -void QGstPipeline::setInStoppedState(bool stopped) +QGstPipeline::QGstPipeline(GstPipeline *p, RefMode mode) : QGstBin(qGstCheckedCast<GstBin>(p), mode) { - Q_ASSERT(d); - d->inStoppedState = stopped; } -void QGstPipeline::setFlushOnConfigChanges(bool flush) -{ - d->m_flushOnConfigChanges = flush; -} +QGstPipeline::~QGstPipeline() = default; void QGstPipeline::installMessageFilter(QGstreamerSyncMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->installMessageFilter(filter); } void QGstPipeline::removeMessageFilter(QGstreamerSyncMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->removeMessageFilter(filter); } void QGstPipeline::installMessageFilter(QGstreamerBusMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->installMessageFilter(filter); } void QGstPipeline::removeMessageFilter(QGstreamerBusMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->removeMessageFilter(filter); } GstStateChangeReturn QGstPipeline::setState(GstState state) { - auto retval = gst_element_set_state(element(), state); - if (d->m_pendingFlush) { - d->m_pendingFlush = false; - flush(); - } - return retval; + return gst_element_set_state(element(), state); +} + +void QGstPipeline::processMessages(GstMessageType types) +{ + QGstPipelinePrivate *d = getPrivate(); + QGstreamerMessage message{ + gst_bus_pop_filtered(d->m_bus, types), + QGstreamerMessage::HasRef, + }; + d->processMessage(message); } void QGstPipeline::beginConfig() { - if (!d) - return; + QGstPipelinePrivate *d = getPrivate(); Q_ASSERT(!isNull()); ++d->m_configCounter; if (d->m_configCounter > 1) return; - d->m_savedState = state(); + GstState state; + GstState pending; + GstStateChangeReturn stateChangeReturn = gst_element_get_state(element(), &state, &pending, 0); + switch (stateChangeReturn) { + case GST_STATE_CHANGE_ASYNC: { + if (state == GST_STATE_PLAYING) { + // playing->paused transition in progress. wait for it to finish + bool stateChangeSuccessful = this->finishStateChange(); + if (!stateChangeSuccessful) + qWarning() << "QGstPipeline::beginConfig: timeout when waiting for state change"; + } + + state = pending; + break; + } + case GST_STATE_CHANGE_FAILURE: { + qDebug() << "QGstPipeline::beginConfig: state change failure"; + dumpGraph("beginConfigFailure"); + break; + } + + case GST_STATE_CHANGE_NO_PREROLL: + case GST_STATE_CHANGE_SUCCESS: + break; + } + + d->m_savedState = state; if (d->m_savedState == GST_STATE_PLAYING) setStateSync(GST_STATE_PAUSED); } void QGstPipeline::endConfig() { - if (!d) - return; + QGstPipelinePrivate *d = getPrivate(); Q_ASSERT(!isNull()); --d->m_configCounter; if (d->m_configCounter) return; - if (d->m_flushOnConfigChanges) - d->m_pendingFlush = true; if (d->m_savedState == GST_STATE_PLAYING) setState(GST_STATE_PLAYING); d->m_savedState = GST_STATE_NULL; @@ -321,60 +282,108 @@ void QGstPipeline::endConfig() void QGstPipeline::flush() { - seek(position(), d->m_rate); + seek(position()); } -bool QGstPipeline::seek(qint64 pos, double rate) +void QGstPipeline::seek(std::chrono::nanoseconds pos, double rate) { - // always adjust the rate, so it can be set before playback starts + using namespace std::chrono_literals; + + QGstPipelinePrivate *d = getPrivate(); + // always adjust the rate, so it can be set before playback starts // setting position needs a loaded media file that's seekable - d->m_rate = rate; - bool success = gst_element_seek(element(), rate, GST_FORMAT_TIME, - GstSeekFlags(GST_SEEK_FLAG_FLUSH), - GST_SEEK_TYPE_SET, pos, - GST_SEEK_TYPE_SET, -1); - if (!success) - return false; + + qCDebug(qLcGstPipeline) << "QGstPipeline::seek to" << pos << "rate:" << rate; + + bool success = (rate > 0) + ? gst_element_seek(element(), rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, pos.count(), GST_SEEK_TYPE_END, 0) + : gst_element_seek(element(), rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_SET, pos.count()); + + if (!success) { + qDebug() << "seek: gst_element_seek failed" << pos; + return; + } d->m_position = pos; - return true; } -bool QGstPipeline::setPlaybackRate(double rate) +void QGstPipeline::seek(std::chrono::nanoseconds pos) +{ + qCDebug(qLcGstPipeline) << "QGstPipeline::seek to" << pos; + seek(pos, getPrivate()->m_rate); +} + +void QGstPipeline::setPlaybackRate(double rate) { + QGstPipelinePrivate *d = getPrivate(); if (rate == d->m_rate) - return false; - seek(position(), rate); - return true; + return; + + d->m_rate = rate; + + qCDebug(qLcGstPipeline) << "QGstPipeline::setPlaybackRate to" << rate; + + applyPlaybackRate(/*instantRateChange =*/true); } double QGstPipeline::playbackRate() const { + QGstPipelinePrivate *d = getPrivate(); return d->m_rate; } -bool QGstPipeline::setPosition(qint64 pos) +void QGstPipeline::applyPlaybackRate(bool instantRateChange) +{ + QGstPipelinePrivate *d = getPrivate(); + + // do not GST_SEEK_FLAG_FLUSH with GST_SEEK_TYPE_NONE + // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3604 + if (instantRateChange && GST_CHECK_VERSION(1, 18, 0)) { + qCDebug(qLcGstPipeline) << "QGstPipeline::applyPlaybackRate instantly"; + bool success = gst_element_seek( + element(), d->m_rate, GST_FORMAT_UNDEFINED, rateChangeSeekFlags, GST_SEEK_TYPE_NONE, + GST_CLOCK_TIME_NONE, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE); + if (!success) + qDebug() << "setPlaybackRate: gst_element_seek failed"; + } else { + seek(position(), d->m_rate); + } +} + +void QGstPipeline::setPosition(std::chrono::nanoseconds pos) { - return seek(pos, d->m_rate); + seek(pos); } -qint64 QGstPipeline::position() const +std::chrono::nanoseconds QGstPipeline::position() const { - gint64 pos; - if (gst_element_query_position(element(), GST_FORMAT_TIME, &pos)) - d->m_position = pos; + QGstPipelinePrivate *d = getPrivate(); + std::optional<std::chrono::nanoseconds> pos = QGstElement::position(); + if (pos) { + d->m_position = *pos; + qCDebug(qLcGstPipeline) << "QGstPipeline::position:" + << std::chrono::round<std::chrono::milliseconds>(*pos); + } else { + qDebug() << "QGstPipeline: failed to query position, using previous position"; + } + return d->m_position; } -qint64 QGstPipeline::duration() const +std::chrono::milliseconds QGstPipeline::positionInMs() const { - gint64 d; - if (!gst_element_query_duration(element(), GST_FORMAT_TIME, &d)) - return 0.; + using namespace std::chrono; + return round<milliseconds>(position()); +} + +QGstPipelinePrivate *QGstPipeline::getPrivate() const +{ + gpointer p = g_object_get_data(qGstCheckedCast<GObject>(object()), "pipeline-private"); + auto *d = reinterpret_cast<QGstPipelinePrivate *>(p); + Q_ASSERT(d); return d; } QT_END_NAMESPACE - -#include "qgstpipeline.moc" - diff --git a/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h b/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h index d1f585cc2..bd711e553 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef qgstpipeline_p_H #define qgstpipeline_p_H @@ -51,10 +15,10 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <QObject> +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtCore/qobject.h> -#include <qgst_p.h> +#include "qgst_p.h" QT_BEGIN_NAMESPACE @@ -77,24 +41,18 @@ class QGstPipelinePrivate; class QGstPipeline : public QGstBin { - QGstPipelinePrivate *d = nullptr; public: constexpr QGstPipeline() = default; - QGstPipeline(const QGstPipeline &o); - QGstPipeline &operator=(const QGstPipeline &o); - QGstPipeline(const char *name); - QGstPipeline(GstPipeline *p); - ~QGstPipeline() override; - - // This is needed to help us avoid sending QVideoFrames or audio buffers to the - // application while we're prerolling the pipeline. - // QMediaPlayer is still in a stopped state, while we put the gstreamer pipeline into a - // Paused state so that we can get the required metadata of the stream and also have a fast - // transition to play. - bool inStoppedState() const; - void setInStoppedState(bool stopped); - - void setFlushOnConfigChanges(bool flush); + QGstPipeline(const QGstPipeline &) = default; + QGstPipeline(QGstPipeline &&) = default; + QGstPipeline &operator=(const QGstPipeline &) = default; + QGstPipeline &operator=(QGstPipeline &&) noexcept = default; + QGstPipeline(GstPipeline *, RefMode mode); + ~QGstPipeline(); + + // installs QGstPipelinePrivate as "pipeline-private" gobject property + static QGstPipeline create(const char *name); + static QGstPipeline adopt(GstPipeline *); void installMessageFilter(QGstreamerSyncMessageFilter *filter); void removeMessageFilter(QGstreamerSyncMessageFilter *filter); @@ -103,36 +61,45 @@ public: GstStateChangeReturn setState(GstState state); - GstPipeline *pipeline() const { return GST_PIPELINE_CAST(m_object); } + GstPipeline *pipeline() const { return GST_PIPELINE_CAST(get()); } + + void processMessages(GstMessageType = GST_MESSAGE_ANY); - void dumpGraph(const char *fileName) + template <typename Functor> + void modifyPipelineWhileNotRunning(Functor &&fn) { - if (isNull()) - return; - -#if 1 //def QT_GST_CAPTURE_DEBUG - GST_DEBUG_BIN_TO_DOT_FILE(bin(), - GstDebugGraphDetails(GST_DEBUG_GRAPH_SHOW_ALL | - GST_DEBUG_GRAPH_SHOW_MEDIA_TYPE | GST_DEBUG_GRAPH_SHOW_NON_DEFAULT_PARAMS | GST_DEBUG_GRAPH_SHOW_STATES), - fileName); -#else - Q_UNUSED(fileName); -#endif + beginConfig(); + fn(); + endConfig(); } - void beginConfig(); - void endConfig(); + template <typename Functor> + static void modifyPipelineWhileNotRunning(QGstPipeline &&pipeline, Functor &&fn) + { + if (pipeline) + pipeline.modifyPipelineWhileNotRunning(fn); + else + fn(); + } void flush(); - bool seek(qint64 pos, double rate); - bool setPlaybackRate(double rate); + void setPlaybackRate(double rate); double playbackRate() const; + void applyPlaybackRate(bool instantRateChange); + + void setPosition(std::chrono::nanoseconds pos); + std::chrono::nanoseconds position() const; + std::chrono::milliseconds positionInMs() const; + +private: + void seek(std::chrono::nanoseconds pos, double rate); + void seek(std::chrono::nanoseconds pos); - bool setPosition(qint64 pos); - qint64 position() const; + QGstPipelinePrivate *getPrivate() const; - qint64 duration() const; + void beginConfig(); + void endConfig(); }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp index 402b3c825..a2f60eaa1 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp @@ -1,148 +1,156 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreameraudioinput_p.h> -#include <qgstreameraudiodevice_p.h> -#include <qaudiodevice.h> -#include <qaudioinput.h> +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreameraudioinput_p.h> #include <QtCore/qloggingcategory.h> -#include <QtNetwork/qnetworkaccessmanager.h> -#include <QtNetwork/qnetworkreply.h> +#include <QtMultimedia/qaudiodevice.h> +#include <QtMultimedia/qaudioinput.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> +#include <audio/qgstreameraudiodevice_p.h> +#include <common/qgstpipeline_p.h> -Q_LOGGING_CATEGORY(qLcMediaAudioInput, "qt.multimedia.audioInput") QT_BEGIN_NAMESPACE -QGstreamerAudioInput::QGstreamerAudioInput(QAudioInput *parent) - : QObject(parent), - QPlatformAudioInput(parent), - gstAudioInput("audioInput") +namespace { + +Q_LOGGING_CATEGORY(qLcMediaAudioInput, "qt.multimedia.audioinput") + +constexpr QLatin1String defaultSrcName = [] { + using namespace Qt::Literals; + + if constexpr (QT_CONFIG(pulseaudio)) + return "pulsesrc"_L1; + else if constexpr (QT_CONFIG(alsa)) + return "alsasrc"_L1; + else + return "autoaudiosrc"_L1; +}(); + +bool hasDeviceProperty(const QGstElement &element) { - audioSrc = QGstElement("autoaudiosrc", "autoaudiosrc"); - audioVolume = QGstElement("volume", "volume"); - gstAudioInput.add(audioSrc, audioVolume); - audioSrc.link(audioVolume); + using namespace Qt::Literals; + QLatin1String elementType = element.typeName(); + + if constexpr (QT_CONFIG(pulseaudio)) + return elementType == "GstPulseSrc"_L1; + + if constexpr (0 && QT_CONFIG(alsa)) // alsasrc has a "device" property, but it cannot be changed + // during playback + return elementType == "GstAlsaSrc"_L1; - gstAudioInput.addGhostPad(audioVolume, "src"); + return false; } -QGstreamerAudioInput::~QGstreamerAudioInput() +} // namespace + +QMaybe<QPlatformAudioInput *> QGstreamerAudioInput::create(QAudioInput *parent) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable("autoaudiosrc", "volume"); + if (error) + return *error; + + return new QGstreamerAudioInput(parent); +} + +QGstreamerAudioInput::QGstreamerAudioInput(QAudioInput *parent) + : QObject(parent), + QPlatformAudioInput(parent), + m_audioInputBin(QGstBin::create("audioInput")), + m_audioSrc{ + QGstElement::createFromFactory(defaultSrcName.constData(), "autoaudiosrc"), + }, + m_audioVolume{ + QGstElement::createFromFactory("volume", "volume"), + } { - gstAudioInput.setStateSync(GST_STATE_NULL); + m_audioInputBin.add(m_audioSrc, m_audioVolume); + qLinkGstElements(m_audioSrc, m_audioVolume); + + m_audioInputBin.addGhostPad(m_audioVolume, "src"); } -int QGstreamerAudioInput::volume() const +QGstElement QGstreamerAudioInput::createGstElement() { - return m_volume; + const auto *customDeviceInfo = + dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(m_audioDevice.handle()); + + if (customDeviceInfo) { + qCDebug(qLcMediaAudioInput) + << "requesting custom audio src element: " << customDeviceInfo->id; + + QGstElement element = QGstBin::createFromPipelineDescription(customDeviceInfo->id, + /*name=*/nullptr, + /*ghostUnlinkedPads=*/true); + if (element) + return element; + + qCWarning(qLcMediaAudioInput) + << "Cannot create audio source element:" << customDeviceInfo->id; + } + + const QByteArray &id = m_audioDevice.id(); + if constexpr (QT_CONFIG(pulseaudio) || QT_CONFIG(alsa)) { + QGstElement newSrc = QGstElement::createFromFactory(defaultSrcName.constData(), "audiosrc"); + if (newSrc) { + newSrc.set("device", id.constData()); + return newSrc; + } + + qWarning() << "Cannot create" << defaultSrcName; + + } else { + auto *deviceInfo = dynamic_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); + if (deviceInfo && deviceInfo->gstDevice) { + QGstElement element = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosrc"); + if (element) + return element; + } + } + qCWarning(qLcMediaAudioInput) << "Invalid audio device"; + qCWarning(qLcMediaAudioInput) + << "Failed to create a gst element for the audio device, using a default audio source"; + return QGstElement::createFromFactory("autoaudiosrc", "audiosrc"); } -bool QGstreamerAudioInput::isMuted() const +QGstreamerAudioInput::~QGstreamerAudioInput() { - return m_muted; + m_audioInputBin.setStateSync(GST_STATE_NULL); } -void QGstreamerAudioInput::setVolume(float vol) +void QGstreamerAudioInput::setVolume(float volume) { - if (vol == m_volume) - return; - m_volume = vol; - audioVolume.set("volume", vol); - emit volumeChanged(m_volume); + m_audioVolume.set("volume", volume); } void QGstreamerAudioInput::setMuted(bool muted) { - if (muted == m_muted) - return; - m_muted = muted; - audioVolume.set("mute", muted); - emit mutedChanged(muted); + m_audioVolume.set("mute", muted); } void QGstreamerAudioInput::setAudioDevice(const QAudioDevice &device) { if (device == m_audioDevice) return; - qCDebug(qLcMediaAudioInput) << "setAudioInput" << device.description() << device.isNull(); + qCDebug(qLcMediaAudioInput) << "setAudioDevice" << device.description() << device.isNull(); m_audioDevice = device; - QGstElement newSrc; -#if QT_CONFIG(pulseaudio) - auto id = m_audioDevice.id(); - newSrc = QGstElement("pulsesrc", "audiosrc"); - if (!newSrc.isNull()) - newSrc.set("device", id.constData()); - else - qCWarning(qLcMediaAudioInput) << "Invalid audio device"; -#else - auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); - if (deviceInfo && deviceInfo->gstDevice) - newSrc = gst_device_create_element(deviceInfo->gstDevice, "audiosrc"); - else - qCWarning(qLcMediaAudioInput) << "Invalid audio device"; -#endif - - if (newSrc.isNull()) { - qCWarning(qLcMediaAudioInput) << "Failed to create a gst element for the audio device, using a default audio source"; - newSrc = QGstElement("autoaudiosrc", "audiosrc"); + if (hasDeviceProperty(m_audioSrc) && !isCustomAudioDevice(m_audioDevice)) { + m_audioSrc.set("device", m_audioDevice.id().constData()); + return; } - // FIXME: most probably source can be disconnected outside of idle probe - audioSrc.staticPad("src").doInIdleProbe([&](){ - audioSrc.unlink(audioVolume); - }); - audioSrc.setStateSync(GST_STATE_NULL); - gstAudioInput.remove(audioSrc); - audioSrc = newSrc; - gstAudioInput.add(audioSrc); - audioSrc.link(audioVolume); - audioSrc.syncStateWithParent(); -} + QGstElement newSrc = createGstElement(); -QAudioDevice QGstreamerAudioInput::audioInput() const -{ - return m_audioDevice; + QGstPipeline::modifyPipelineWhileNotRunning(m_audioInputBin.getPipeline(), [&] { + qUnlinkGstElements(m_audioSrc, m_audioVolume); + m_audioInputBin.stopAndRemoveElements(m_audioSrc); + m_audioSrc = std::move(newSrc); + m_audioInputBin.add(m_audioSrc); + qLinkGstElements(m_audioSrc, m_audioVolume); + m_audioSrc.syncStateWithParent(); + }); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h index 3b4ad3475..4b01b53a6 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIOINPUT_P_H #define QGSTREAMERAUDIOINPUT_P_H @@ -51,55 +15,39 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <qaudiodevice.h> - #include <QtCore/qobject.h> +#include <QtMultimedia/private/qplatformaudioinput_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> -#include <private/qplatformaudioinput_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QGstreamerMessage; class QAudioDevice; -class Q_MULTIMEDIA_EXPORT QGstreamerAudioInput : public QObject, public QPlatformAudioInput +class QGstreamerAudioInput : public QObject, public QPlatformAudioInput { - Q_OBJECT - public: - QGstreamerAudioInput(QAudioInput *parent); + static QMaybe<QPlatformAudioInput *> create(QAudioInput *parent); ~QGstreamerAudioInput(); - int volume() const; - bool isMuted() const; - - bool setAudioInput(const QAudioDevice &); - QAudioDevice audioInput() const; - void setAudioDevice(const QAudioDevice &) override; - void setVolume(float volume) override; - void setMuted(bool muted) override; - - QGstElement gstElement() const { return gstAudioInput; } + void setVolume(float) override; + void setMuted(bool) override; -Q_SIGNALS: - void mutedChanged(bool); - void volumeChanged(int); + QGstElement gstElement() const { return m_audioInputBin; } private: - float m_volume = 1.; - bool m_muted = false; + explicit QGstreamerAudioInput(QAudioInput *parent); + + QGstElement createGstElement(); QAudioDevice m_audioDevice; // Gst elements - QGstBin gstAudioInput; + QGstBin m_audioInputBin; - QGstElement audioSrc; - QGstElement audioVolume; + QGstElement m_audioSrc; + QGstElement m_audioVolume; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp index 0c9f84fbc..1a8c6976c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp @@ -1,133 +1,170 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreameraudiooutput_p.h> -#include <qgstreameraudiodevice_p.h> -#include <qaudiodevice.h> -#include <qaudiooutput.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreameraudiooutput_p.h> #include <QtCore/qloggingcategory.h> -#include <QtNetwork/qnetworkaccessmanager.h> -#include <QtNetwork/qnetworkreply.h> +#include <QtMultimedia/qaudiodevice.h> +#include <QtMultimedia/qaudiooutput.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> +#include <common/qgstpipeline_p.h> +#include <audio/qgstreameraudiodevice_p.h> -Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput") QT_BEGIN_NAMESPACE +namespace { + +Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput") + +constexpr QLatin1String defaultSinkName = [] { + using namespace Qt::Literals; + + if constexpr (QT_CONFIG(pulseaudio)) + return "pulsesink"_L1; + else if constexpr (QT_CONFIG(alsa)) + return "alsasink"_L1; + else + return "autoaudiosink"_L1; +}(); + +bool hasDeviceProperty(const QGstElement &element) +{ + using namespace Qt::Literals; + QLatin1String elementType = element.typeName(); + + if constexpr (QT_CONFIG(pulseaudio)) + return elementType == "GstPulseSink"_L1; + if constexpr (0 && QT_CONFIG(alsa)) // alsasrc has a "device" property, but it cannot be changed + // during playback + return elementType == "GstAlsaSink"_L1; + + return false; +} + +} // namespace + +QMaybe<QPlatformAudioOutput *> QGstreamerAudioOutput::create(QAudioOutput *parent) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "audioconvert", "audioresample", "volume", "autoaudiosink"); + if (error) + return *error; + + return new QGstreamerAudioOutput(parent); +} + QGstreamerAudioOutput::QGstreamerAudioOutput(QAudioOutput *parent) - : QObject(parent), - QPlatformAudioOutput(parent), - gstAudioOutput("audioOutput") + : QObject(parent), + QPlatformAudioOutput(parent), + m_audioOutputBin(QGstBin::create("audioOutput")), + m_audioQueue{ + QGstElement::createFromFactory("queue", "audioQueue"), + }, + m_audioConvert{ + QGstElement::createFromFactory("audioconvert", "audioConvert"), + }, + m_audioResample{ + QGstElement::createFromFactory("audioresample", "audioResample"), + }, + m_audioVolume{ + QGstElement::createFromFactory("volume", "volume"), + }, + m_audioSink{ + QGstElement::createFromFactory(defaultSinkName.constData(), "audiosink"), + } { - audioQueue = QGstElement("queue", "audioQueue"); - audioConvert = QGstElement("audioconvert", "audioConvert"); - audioResample = QGstElement("audioresample", "audioResample"); - audioVolume = QGstElement("volume", "volume"); - audioSink = QGstElement("autoaudiosink", "autoAudioSink"); - gstAudioOutput.add(audioQueue, audioConvert, audioResample, audioVolume, audioSink); - audioQueue.link(audioConvert, audioResample, audioVolume, audioSink); - - gstAudioOutput.addGhostPad(audioQueue, "sink"); + m_audioOutputBin.add(m_audioQueue, m_audioConvert, m_audioResample, m_audioVolume, m_audioSink); + qLinkGstElements(m_audioQueue, m_audioConvert, m_audioResample, m_audioVolume, m_audioSink); + + m_audioOutputBin.addGhostPad(m_audioQueue, "sink"); } -QGstreamerAudioOutput::~QGstreamerAudioOutput() +QGstElement QGstreamerAudioOutput::createGstElement() { - gstAudioOutput.setStateSync(GST_STATE_NULL); + const auto *customDeviceInfo = + dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(m_audioDevice.handle()); + + if (customDeviceInfo) { + qCDebug(qLcMediaAudioOutput) + << "requesting custom audio sink element: " << customDeviceInfo->id; + + QGstElement element = + QGstBin::createFromPipelineDescription(customDeviceInfo->id, /*name=*/nullptr, + /*ghostUnlinkedPads=*/true); + if (element) + return element; + + qCWarning(qLcMediaAudioOutput) + << "Cannot create audio sink element:" << customDeviceInfo->id; + } + + const QByteArray &id = m_audioDevice.id(); + if constexpr (QT_CONFIG(pulseaudio) || QT_CONFIG(alsa)) { + QGstElement newSink = + QGstElement::createFromFactory(defaultSinkName.constData(), "audiosink"); + if (newSink) { + newSink.set("device", id.constData()); + return newSink; + } + + qWarning() << "Cannot create" << defaultSinkName; + } else { + auto *deviceInfo = dynamic_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); + if (deviceInfo && deviceInfo->gstDevice) { + QGstElement element = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosink"); + if (element) + return element; + } + } + qCWarning(qLcMediaAudioOutput) << "Invalid audio device:" << m_audioDevice.id(); + qCWarning(qLcMediaAudioOutput) + << "Failed to create a gst element for the audio device, using a default audio sink"; + return QGstElement::createFromFactory("autoaudiosink", "audiosink"); } -void QGstreamerAudioOutput::setVolume(float vol) +QGstreamerAudioOutput::~QGstreamerAudioOutput() { - audioVolume.set("volume", vol); + m_audioOutputBin.setStateSync(GST_STATE_NULL); } -void QGstreamerAudioOutput::setMuted(bool muted) +void QGstreamerAudioOutput::setVolume(float volume) { - audioVolume.set("mute", muted); + m_audioVolume.set("volume", volume); } -void QGstreamerAudioOutput::setPipeline(const QGstPipeline &pipeline) +void QGstreamerAudioOutput::setMuted(bool muted) { - gstPipeline = pipeline; + m_audioVolume.set("mute", muted); } -void QGstreamerAudioOutput::setAudioDevice(const QAudioDevice &info) +void QGstreamerAudioOutput::setAudioDevice(const QAudioDevice &device) { - if (info == m_audioOutput) + if (device == m_audioDevice) return; - qCDebug(qLcMediaAudioOutput) << "setAudioOutput" << info.description() << info.isNull(); - m_audioOutput = info; - - QGstElement newSink; -#if QT_CONFIG(pulseaudio) - auto id = m_audioOutput.id(); - newSink = QGstElement("pulsesink", "audiosink"); - if (!newSink.isNull()) - newSink.set("device", id.constData()); - else - qCWarning(qLcMediaAudioOutput) << "Invalid audio device"; -#else - auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_audioOutput.handle()); - if (deviceInfo && deviceInfo->gstDevice) - newSink = gst_device_create_element(deviceInfo->gstDevice , "audiosink"); - else - qCWarning(qLcMediaAudioOutput) << "Invalid audio device"; -#endif + qCDebug(qLcMediaAudioOutput) << "setAudioDevice" << device.description() << device.isNull(); + + m_audioDevice = device; - if (newSink.isNull()) { - qCWarning(qLcMediaAudioOutput) << "Failed to create a gst element for the audio device, using a default audio sink"; - newSink = QGstElement("autoaudiosink", "audiosink"); + if (hasDeviceProperty(m_audioSink) && !isCustomAudioDevice(m_audioDevice)) { + m_audioSink.set("device", m_audioDevice.id().constData()); + return; } - audioVolume.staticPad("src").doInIdleProbe([&](){ - audioVolume.unlink(audioSink); - gstAudioOutput.remove(audioSink); - gstAudioOutput.add(newSink); - newSink.syncStateWithParent(); - audioVolume.link(newSink); + QGstElement newSink = createGstElement(); + + QGstPipeline::modifyPipelineWhileNotRunning(m_audioOutputBin.getPipeline(), [&] { + qUnlinkGstElements(m_audioVolume, m_audioSink); + m_audioOutputBin.stopAndRemoveElements(m_audioSink); + m_audioSink = std::move(newSink); + m_audioOutputBin.add(m_audioSink); + m_audioSink.syncStateWithParent(); + qLinkGstElements(m_audioVolume, m_audioSink); }); - audioSink.setStateSync(GST_STATE_NULL); - audioSink = newSink; + // we need to flush the pipeline, otherwise, the new sink doesn't always reach the new state + if (m_audioOutputBin.getPipeline()) + m_audioOutputBin.getPipeline().flush(); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h index 318419247..da11c39d2 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIOOUTPUT_P_H #define QGSTREAMERAUDIOOUTPUT_P_H @@ -51,52 +15,42 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <qaudiodevice.h> - #include <QtCore/qobject.h> +#include <QtMultimedia/private/qplatformaudiooutput_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> -#include <private/qplatformaudiooutput_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QGstreamerMessage; class QAudioDevice; -class Q_MULTIMEDIA_EXPORT QGstreamerAudioOutput : public QObject, public QPlatformAudioOutput +class QGstreamerAudioOutput : public QObject, public QPlatformAudioOutput { - Q_OBJECT - public: - QGstreamerAudioOutput(QAudioOutput *parent); + static QMaybe<QPlatformAudioOutput *> create(QAudioOutput *parent); ~QGstreamerAudioOutput(); void setAudioDevice(const QAudioDevice &) override; - void setVolume(float volume) override; - void setMuted(bool muted) override; + void setVolume(float) override; + void setMuted(bool) override; - void setPipeline(const QGstPipeline &pipeline); + QGstElement gstElement() const { return m_audioOutputBin; } - QGstElement gstElement() const { return gstAudioOutput; } +private: + explicit QGstreamerAudioOutput(QAudioOutput *parent); -Q_SIGNALS: - void mutedChanged(bool); - void volumeChanged(int); + QGstElement createGstElement(); -private: - QAudioDevice m_audioOutput; + QAudioDevice m_audioDevice; // Gst elements - QGstPipeline gstPipeline; - QGstBin gstAudioOutput; - - QGstElement audioQueue; - QGstElement audioConvert; - QGstElement audioResample; - QGstElement audioVolume; - QGstElement audioSink; + QGstBin m_audioOutputBin; + + QGstElement m_audioQueue; + QGstElement m_audioConvert; + QGstElement m_audioResample; + QGstElement m_audioVolume; + QGstElement m_audioSink; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp index 3af8000bb..9cba810db 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp @@ -1,44 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamerbufferprobe_p.h" -#include "qgstutils_p.h" +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreamerbufferprobe_p.h> + +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE @@ -51,10 +16,14 @@ QGstreamerBufferProbe::~QGstreamerBufferProbe() = default; void QGstreamerBufferProbe::addProbeToPad(GstPad *pad, bool downstream) { - if (GstCaps *caps = gst_pad_get_current_caps(pad)) { - probeCaps(caps); - gst_caps_unref(caps); - } + QGstCaps caps{ + gst_pad_get_current_caps(pad), + QGstCaps::HasRef, + }; + + if (caps) + probeCaps(caps.caps()); + if (m_flags & ProbeCaps) { m_capsProbeId = gst_pad_add_probe( pad, diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h index d3ca9db2e..71996a0cc 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERBUFFERPROBE_H #define QGSTREAMERBUFFERPROBE_H @@ -59,7 +23,7 @@ QT_BEGIN_NAMESPACE -class Q_MULTIMEDIA_EXPORT QGstreamerBufferProbe +class QGstreamerBufferProbe { public: enum Flags diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp index c8e730c3d..e4b3d7393 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp @@ -1,73 +1,41 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreamermediaplayer_p.h> -#include <qgstpipeline_p.h> -#include <qgstreamermetadata_p.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreamermediaplayer_p.h> + +#include <audio/qgstreameraudiodevice_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstappsource_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreameraudiooutput_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgstreamermetadata_p.h> +#include <common/qgstreamervideooutput_p.h> +#include <common/qgstreamervideosink_p.h> #include <qgstreamerformatinfo_p.h> -#include <qgstreameraudiooutput_p.h> -#include <qgstreamervideooutput_p.h> -#include <qgstreamervideosink_p.h> -#include "qgstreamermessage_p.h" -#include <qgstreameraudiodevice_p.h> -#include <qgstappsrc_p.h> -#include <qaudiodevice.h> +#include <QtMultimedia/qaudiodevice.h> #include <QtCore/qdir.h> #include <QtCore/qsocketnotifier.h> #include <QtCore/qurl.h> #include <QtCore/qdebug.h> #include <QtCore/qloggingcategory.h> -#include <QtNetwork/qnetworkaccessmanager.h> -#include <QtNetwork/qnetworkreply.h> +#include <QtCore/private/quniquehandle_p.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> -Q_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player") +#if QT_CONFIG(gstreamer_gl) +# include <gst/gl/gl.h> +#endif + +static Q_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player") QT_BEGIN_NAMESPACE -QGstreamerMediaPlayer::TrackSelector::TrackSelector(TrackType type, const char *name) - : selector(QGstElement("input-selector", name)), - type(type) +QGstreamerMediaPlayer::TrackSelector::TrackSelector(TrackType type, QGstElement selector) + : selector(selector), type(type) { selector.set("sync-streams", true); selector.set("sync-mode", 1 /*clock*/); @@ -110,35 +78,87 @@ QGstreamerMediaPlayer::TrackSelector &QGstreamerMediaPlayer::trackSelector(Track return ts; } -QGstreamerMediaPlayer::QGstreamerMediaPlayer(QMediaPlayer *parent) +void QGstreamerMediaPlayer::mediaStatusChanged(QMediaPlayer::MediaStatus status) +{ + if (status != QMediaPlayer::StalledMedia) + m_stalledMediaNotifier.stop(); + + QPlatformMediaPlayer::mediaStatusChanged(status); +} + +void QGstreamerMediaPlayer::updateBufferProgress(float newProgress) +{ + if (qFuzzyIsNull(newProgress - m_bufferProgress)) + return; + + m_bufferProgress = newProgress; + bufferProgressChanged(m_bufferProgress); +} + +void QGstreamerMediaPlayer::disconnectDecoderHandlers() +{ + auto handlers = std::initializer_list<QGObjectHandlerScopedConnection *>{ + &padAdded, &padRemoved, &sourceSetup, &uridecodebinElementAdded, + &unknownType, &elementAdded, &elementRemoved, + }; + for (QGObjectHandlerScopedConnection *handler : handlers) + handler->disconnect(); + + decodeBinQueues = 0; +} + +QMaybe<QPlatformMediaPlayer *> QGstreamerMediaPlayer::create(QMediaPlayer *parent) +{ + auto videoOutput = QGstreamerVideoOutput::create(); + if (!videoOutput) + return videoOutput.error(); + + static const auto error = + qGstErrorMessageIfElementsNotAvailable("input-selector", "decodebin", "uridecodebin"); + if (error) + return *error; + + return new QGstreamerMediaPlayer(videoOutput.value(), parent); +} + +QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, + QMediaPlayer *parent) : QObject(parent), QPlatformMediaPlayer(parent), - trackSelectors{{{ VideoStream, "videoInputSelector" }, - { AudioStream, "audioInputSelector" }, - { SubtitleStream, "subTitleInputSelector" }}}, - playerPipeline("playerPipeline") + trackSelectors{ { + { VideoStream, + QGstElement::createFromFactory("input-selector", "videoInputSelector") }, + { AudioStream, + QGstElement::createFromFactory("input-selector", "audioInputSelector") }, + { SubtitleStream, + QGstElement::createFromFactory("input-selector", "subTitleInputSelector") }, + } }, + playerPipeline(QGstPipeline::create("playerPipeline")), + gstVideoOutput(videoOutput) { - playerPipeline.setFlushOnConfigChanges(true); - - gstVideoOutput = new QGstreamerVideoOutput(this); + gstVideoOutput->setParent(this); gstVideoOutput->setPipeline(playerPipeline); for (auto &ts : trackSelectors) playerPipeline.add(ts.selector); - playerPipeline.setState(GST_STATE_NULL); playerPipeline.installMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this)); playerPipeline.installMessageFilter(static_cast<QGstreamerSyncMessageFilter *>(this)); - gst_pipeline_use_clock(playerPipeline.pipeline(), gst_system_clock_obtain()); + QGstClockHandle systemClock{ + gst_system_clock_obtain(), + }; + + gst_pipeline_use_clock(playerPipeline.pipeline(), systemClock.get()); + + connect(&positionUpdateTimer, &QTimer::timeout, this, [this] { + updatePositionFromPipeline(); + }); - /* Taken from gstdicoverer.c: - * This is ugly. We get the GType of decodebin so we can quickly detect - * when a decodebin is added to uridecodebin so we can set the - * post-stream-topology setting to TRUE */ - auto decodebin = QGstElement("decodebin", nullptr); - decodebinType = G_OBJECT_TYPE(decodebin.element()); - connect(&positionUpdateTimer, &QTimer::timeout, this, &QGstreamerMediaPlayer::updatePosition); + m_stalledMediaNotifier.setSingleShot(true); + connect(&m_stalledMediaNotifier, &QTimer::timeout, this, [this] { + mediaStatusChanged(QMediaPlayer::StalledMedia); + }); } QGstreamerMediaPlayer::~QGstreamerMediaPlayer() @@ -146,25 +166,45 @@ QGstreamerMediaPlayer::~QGstreamerMediaPlayer() playerPipeline.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this)); playerPipeline.removeMessageFilter(static_cast<QGstreamerSyncMessageFilter *>(this)); playerPipeline.setStateSync(GST_STATE_NULL); - topology.free(); } -qint64 QGstreamerMediaPlayer::position() const +std::chrono::nanoseconds QGstreamerMediaPlayer::pipelinePosition() const { - if (playerPipeline.isNull() || m_url.isEmpty()) - return 0; + if (!hasMedia()) + return {}; - return playerPipeline.position()/1e6; + Q_ASSERT(playerPipeline); + return playerPipeline.position(); +} + +void QGstreamerMediaPlayer::updatePositionFromPipeline() +{ + using namespace std::chrono; + + positionChanged(round<milliseconds>(pipelinePosition())); +} + +void QGstreamerMediaPlayer::updateDurationFromPipeline() +{ + std::optional<std::chrono::milliseconds> duration = playerPipeline.durationInMs(); + if (!duration) + duration = std::chrono::milliseconds{ -1 }; + + if (duration != m_duration) { + qCDebug(qLcMediaPlayer) << "updateDurationFromPipeline" << *duration; + m_duration = *duration; + durationChanged(m_duration); + } } qint64 QGstreamerMediaPlayer::duration() const { - return m_duration; + return m_duration.count(); } float QGstreamerMediaPlayer::bufferProgress() const { - return m_bufferProgress/100.; + return m_bufferProgress; } QMediaTimeRange QGstreamerMediaPlayer::availablePlaybackRanges() const @@ -179,18 +219,28 @@ qreal QGstreamerMediaPlayer::playbackRate() const void QGstreamerMediaPlayer::setPlaybackRate(qreal rate) { - if (playerPipeline.setPlaybackRate(rate)) - playbackRateChanged(rate); + if (rate == m_rate) + return; + + m_rate = rate; + + playerPipeline.setPlaybackRate(rate); + playbackRateChanged(rate); } void QGstreamerMediaPlayer::setPosition(qint64 pos) { - qint64 currentPos = playerPipeline.position()/1e6; - if (pos == currentPos) + std::chrono::milliseconds posInMs{ pos }; + setPosition(posInMs); +} + +void QGstreamerMediaPlayer::setPosition(std::chrono::milliseconds pos) +{ + if (pos == playerPipeline.position()) return; playerPipeline.finishStateChange(); - playerPipeline.setPosition(pos*1e6); - qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << pos << playerPipeline.position()/1e6; + playerPipeline.setPosition(pos); + qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << pos << playerPipeline.positionInMs(); if (mediaStatus() == QMediaPlayer::EndOfMedia) mediaStatusChanged(QMediaPlayer::LoadedMedia); positionChanged(pos); @@ -198,116 +248,199 @@ void QGstreamerMediaPlayer::setPosition(qint64 pos) void QGstreamerMediaPlayer::play() { - if (state() == QMediaPlayer::PlayingState || m_url.isEmpty()) + QMediaPlayer::PlaybackState currentState = state(); + if (currentState == QMediaPlayer::PlayingState || !hasMedia()) return; - resetCurrentLoop(); - playerPipeline.setInStoppedState(false); + if (currentState != QMediaPlayer::PausedState) + resetCurrentLoop(); + + gstVideoOutput->setActive(true); if (mediaStatus() == QMediaPlayer::EndOfMedia) { - playerPipeline.setPosition(0); - updatePosition(); + playerPipeline.setPosition({}); + positionChanged(0); } qCDebug(qLcMediaPlayer) << "play()."; int ret = playerPipeline.setState(GST_STATE_PLAYING); if (m_requiresSeekOnPlay) { - // Flushing the pipeline is required to get track changes - // immediately, when they happen while paused. + // Flushing the pipeline is required to get track changes immediately, when they happen + // while paused. playerPipeline.flush(); m_requiresSeekOnPlay = false; + } else { + if (currentState == QMediaPlayer::StoppedState) { + // we get an assertion failure during instant playback rate changes + // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3545 + constexpr bool performInstantRateChange = false; + playerPipeline.applyPlaybackRate(/*instantRateChange=*/performInstantRateChange); + } } if (ret == GST_STATE_CHANGE_FAILURE) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the playing state."; - if (mediaStatus() == QMediaPlayer::LoadedMedia) - mediaStatusChanged(QMediaPlayer::BufferedMedia); - emit stateChanged(QMediaPlayer::PlayingState); + positionUpdateTimer.start(100); + stateChanged(QMediaPlayer::PlayingState); } void QGstreamerMediaPlayer::pause() { - if (state() == QMediaPlayer::PausedState || m_url.isEmpty()) + if (state() == QMediaPlayer::PausedState || !hasMedia() + || m_resourceErrorState != ResourceErrorState::NoError) return; positionUpdateTimer.stop(); - if (playerPipeline.inStoppedState()) { - playerPipeline.setInStoppedState(false); - playerPipeline.flush(); - } - int ret = playerPipeline.setState(GST_STATE_PAUSED); + + gstVideoOutput->setActive(true); + int ret = playerPipeline.setStateSync(GST_STATE_PAUSED); if (ret == GST_STATE_CHANGE_FAILURE) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; if (mediaStatus() == QMediaPlayer::EndOfMedia) { - playerPipeline.setPosition(0); - mediaStatusChanged(QMediaPlayer::BufferedMedia); + playerPipeline.setPosition({}); + positionChanged(0); + } else { + updatePositionFromPipeline(); } - updatePosition(); - emit stateChanged(QMediaPlayer::PausedState); + stateChanged(QMediaPlayer::PausedState); + + if (m_bufferProgress > 0 || !canTrackProgress()) + mediaStatusChanged(QMediaPlayer::BufferedMedia); + else + mediaStatusChanged(QMediaPlayer::BufferingMedia); } void QGstreamerMediaPlayer::stop() { - if (state() == QMediaPlayer::StoppedState) + using namespace std::chrono_literals; + if (state() == QMediaPlayer::StoppedState) { + if (position() != 0) { + playerPipeline.setPosition({}); + positionChanged(0ms); + mediaStatusChanged(QMediaPlayer::LoadedMedia); + } return; + } stopOrEOS(false); } +const QGstPipeline &QGstreamerMediaPlayer::pipeline() const +{ + return playerPipeline; +} + void QGstreamerMediaPlayer::stopOrEOS(bool eos) { + using namespace std::chrono_literals; + positionUpdateTimer.stop(); - playerPipeline.setInStoppedState(true); + gstVideoOutput->setActive(false); bool ret = playerPipeline.setStateSync(GST_STATE_PAUSED); if (!ret) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the stopped state."; - if (!eos) - playerPipeline.setPosition(0); - updatePosition(); - emit stateChanged(QMediaPlayer::StoppedState); - mediaStatusChanged(eos ? QMediaPlayer::EndOfMedia : QMediaPlayer::LoadedMedia); + if (!eos) { + playerPipeline.setPosition(0ms); + positionChanged(0ms); + } + stateChanged(QMediaPlayer::StoppedState); + if (eos) + mediaStatusChanged(QMediaPlayer::EndOfMedia); + else + mediaStatusChanged(QMediaPlayer::LoadedMedia); + m_initialBufferProgressSent = false; + bufferProgressChanged(0.f); } -bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) +void QGstreamerMediaPlayer::detectPipelineIsSeekable() { - if (message.isNull()) - return false; + std::optional<bool> canSeek = playerPipeline.canSeek(); + if (canSeek) { + qCDebug(qLcMediaPlayer) << "detectPipelineIsSeekable: pipeline is seekable:" << *canSeek; + seekableChanged(*canSeek); + } else { + qCWarning(qLcMediaPlayer) << "detectPipelineIsSeekable: query for seekable failed."; + seekableChanged(false); + } +} -// qCDebug(qLcMediaPlayer) << "received bus message from" << message.source().name() << message.type() << (message.type() == GST_MESSAGE_TAG); +QGstElement QGstreamerMediaPlayer::getSinkElementForTrackType(TrackType trackType) +{ + switch (trackType) { + case AudioStream: + return gstAudioOutput ? gstAudioOutput->gstElement() : QGstElement{}; + case VideoStream: + return gstVideoOutput ? gstVideoOutput->gstElement() : QGstElement{}; + case SubtitleStream: + return gstVideoOutput ? gstVideoOutput->gstSubtitleElement() : QGstElement{}; + break; + default: + Q_UNREACHABLE_RETURN(QGstElement{}); + } +} - GstMessage* gm = message.rawMessage(); +bool QGstreamerMediaPlayer::hasMedia() const +{ + return !m_url.isEmpty() || m_stream; +} + +bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) +{ + qCDebug(qLcMediaPlayer) << "received bus message:" << message; + + GstMessage* gm = message.message(); switch (message.type()) { case GST_MESSAGE_TAG: { // #### This isn't ideal. We shouldn't catch stream specific tags here, rather the global ones - GstTagList *tag_list; - gst_message_parse_tag(gm, &tag_list); - //qCDebug(qLcMediaPlayer) << "Got tags: " << message.source().name() << gst_tag_list_to_string(tag_list); - auto metaData = QGstreamerMetaData::fromGstTagList(tag_list); - for (auto k : metaData.keys()) - m_metaData.insert(k, metaData.value(k)); + QGstTagListHandle tagList; + gst_message_parse_tag(gm, &tagList); + + qCDebug(qLcMediaPlayer) << " Got tags: " << tagList.get(); + + QMediaMetaData originalMetaData = m_metaData; + extendMetaDataFromTagList(m_metaData, tagList); + if (originalMetaData != m_metaData) + metaDataChanged(); + + if (gstVideoOutput) { + QVariant rotation = m_metaData.value(QMediaMetaData::Orientation); + gstVideoOutput->setRotation(rotation.value<QtVideo::Rotation>()); + } break; } case GST_MESSAGE_DURATION_CHANGED: { - qint64 d = playerPipeline.duration()/1e6; - qCDebug(qLcMediaPlayer) << " duration changed message" << d; - if (d != m_duration) { - m_duration = d; - emit durationChanged(duration()); - } + if (!prerolling) + updateDurationFromPipeline(); + return false; } - case GST_MESSAGE_EOS: + case GST_MESSAGE_EOS: { + positionChanged(m_duration); if (doLoop()) { setPosition(0); break; } stopOrEOS(true); break; + } case GST_MESSAGE_BUFFERING: { - qCDebug(qLcMediaPlayer) << " buffering message"; int progress = 0; gst_message_parse_buffering(gm, &progress); - m_bufferProgress = progress; - mediaStatusChanged(m_bufferProgress == 100 ? QMediaPlayer::BufferedMedia : QMediaPlayer::BufferingMedia); - emit bufferProgressChanged(m_bufferProgress/100.); + + if (state() != QMediaPlayer::StoppedState && !prerolling) { + if (!m_initialBufferProgressSent) { + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + } + + if (m_bufferProgress > 0 && progress == 0) { + m_stalledMediaNotifier.start(stalledMediaDebouncePeriod); + } else if (progress >= 50) + // QTBUG-124517: rethink buffering + mediaStatusChanged(QMediaPlayer::BufferedMedia); + else + mediaStatusChanged(QMediaPlayer::BufferingMedia); + } + + updateBufferProgress(progress * 0.01); break; } case GST_MESSAGE_STATE_CHANGED: { @@ -319,116 +452,126 @@ bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) GstState pending; gst_message_parse_state_changed(gm, &oldState, &newState, &pending); - qCDebug(qLcMediaPlayer) << " state changed message" << oldState << newState << pending; - -#ifdef DEBUG_PLAYBIN - static QStringList states = { - QStringLiteral("GST_STATE_VOID_PENDING"), QStringLiteral("GST_STATE_NULL"), - QStringLiteral("GST_STATE_READY"), QStringLiteral("GST_STATE_PAUSED"), - QStringLiteral("GST_STATE_PLAYING") }; - - qCDebug(qLcMediaPlayer) << QStringLiteral("state changed: old: %1 new: %2 pending: %3") \ - .arg(states[oldState]) \ - .arg(states[newState]) \ - .arg(states[pending]); -#endif + qCDebug(qLcMediaPlayer) << " state changed message from" + << QCompactGstMessageAdaptor(message); switch (newState) { case GST_STATE_VOID_PENDING: case GST_STATE_NULL: case GST_STATE_READY: break; - case GST_STATE_PAUSED: - { + case GST_STATE_PAUSED: { if (prerolling) { qCDebug(qLcMediaPlayer) << "Preroll done, setting status to Loaded"; + playerPipeline.dumpGraph("playerPipelinePrerollDone"); + prerolling = false; - GST_DEBUG_BIN_TO_DOT_FILE(playerPipeline.bin(), GST_DEBUG_GRAPH_SHOW_ALL, "playerPipeline"); - qint64 d = playerPipeline.duration()/1e6; - if (d != m_duration) { - m_duration = d; - qCDebug(qLcMediaPlayer) << " duration changed" << d; - emit durationChanged(duration()); - } + updateDurationFromPipeline(); + m_metaData.insert(QMediaMetaData::Duration, duration()); + if (!m_url.isEmpty()) + m_metaData.insert(QMediaMetaData::Url, m_url); parseStreamsAndMetadata(); + metaDataChanged(); - emit tracksChanged(); + tracksChanged(); mediaStatusChanged(QMediaPlayer::LoadedMedia); - GstQuery *query = gst_query_new_seeking(GST_FORMAT_TIME); - gboolean canSeek = false; - if (gst_element_query(playerPipeline.element(), query)) { - gst_query_parse_seeking(query, NULL, &canSeek, nullptr, nullptr); - qCDebug(qLcMediaPlayer) << " pipeline is seekable:" << canSeek; - } else { - qCDebug(qLcMediaPlayer) << " query for seekable failed."; + if (state() == QMediaPlayer::PlayingState) { + Q_ASSERT(!m_initialBufferProgressSent); + + bool immediatelySendBuffered = !canTrackProgress() || m_bufferProgress > 0; + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + if (immediatelySendBuffered) + mediaStatusChanged(QMediaPlayer::BufferedMedia); } - gst_query_unref(query); - seekableChanged(canSeek); } break; } - case GST_STATE_PLAYING: - mediaStatusChanged(QMediaPlayer::BufferedMedia); + case GST_STATE_PLAYING: { + if (!m_initialBufferProgressSent) { + bool immediatelySendBuffered = !canTrackProgress() || m_bufferProgress > 0; + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + if (immediatelySendBuffered) + mediaStatusChanged(QMediaPlayer::BufferedMedia); + } break; } + } break; } case GST_MESSAGE_ERROR: { - GError *err; - gchar *debug; + qCDebug(qLcMediaPlayer) << " error" << QCompactGstMessageAdaptor(message); + + QUniqueGErrorHandle err; + QUniqueGStringHandle debug; gst_message_parse_error(gm, &err, &debug); - if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) - emit error(QMediaPlayer::FormatError, tr("Cannot play stream of type: <unknown>")); - else - emit error(QMediaPlayer::ResourceError, QString::fromUtf8(err->message)); - playerPipeline.dumpGraph("error"); + GQuark errorDomain = err.get()->domain; + gint errorCode = err.get()->code; + + if (errorDomain == GST_STREAM_ERROR) { + if (errorCode == GST_STREAM_ERROR_CODEC_NOT_FOUND) + error(QMediaPlayer::FormatError, tr("Cannot play stream of type: <unknown>")); + else { + error(QMediaPlayer::FormatError, QString::fromUtf8(err.get()->message)); + } + } else if (errorDomain == GST_RESOURCE_ERROR) { + if (errorCode == GST_RESOURCE_ERROR_NOT_FOUND) { + if (m_resourceErrorState != ResourceErrorState::ErrorReported) { + // gstreamer seems to deliver multiple GST_RESOURCE_ERROR_NOT_FOUND events + error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); + m_resourceErrorState = ResourceErrorState::ErrorReported; + m_url.clear(); + m_stream = nullptr; + } + } else { + error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); + } + } else { + playerPipeline.dumpGraph("error"); + } mediaStatusChanged(QMediaPlayer::InvalidMedia); - g_error_free(err); - g_free(debug); break; } - case GST_MESSAGE_WARNING: { - GError *err; - gchar *debug; - gst_message_parse_warning (gm, &err, &debug); - qCWarning(qLcMediaPlayer) << "Warning:" << QString::fromUtf8(err->message); + + case GST_MESSAGE_WARNING: + qCWarning(qLcMediaPlayer) << "Warning:" << QCompactGstMessageAdaptor(message); playerPipeline.dumpGraph("warning"); - g_error_free (err); - g_free (debug); break; - } - case GST_MESSAGE_INFO: { - if (qLcMediaPlayer().isDebugEnabled()) { - GError *err; - gchar *debug; - gst_message_parse_info (gm, &err, &debug); - qCDebug(qLcMediaPlayer) << "Info:" << QString::fromUtf8(err->message); - g_error_free (err); - g_free (debug); - } + + case GST_MESSAGE_INFO: + if (qLcMediaPlayer().isDebugEnabled()) + qCDebug(qLcMediaPlayer) << "Info:" << QCompactGstMessageAdaptor(message); break; - } + case GST_MESSAGE_SEGMENT_START: { qCDebug(qLcMediaPlayer) << " segment start message, updating position"; - QGstStructure structure(gst_message_get_structure(gm)); + QGstStructureView structure(gst_message_get_structure(gm)); auto p = structure["position"].toInt64(); if (p) { - qint64 position = (*p)/1000000; - emit positionChanged(position); + std::chrono::milliseconds position{ + (*p) / 1000000, + }; + positionChanged(position); } break; } case GST_MESSAGE_ELEMENT: { - QGstStructure structure(gst_message_get_structure(gm)); + QGstStructureView structure(gst_message_get_structure(gm)); auto type = structure.name(); - if (type == "stream-topology") { - topology.free(); - topology = structure.copy(); - } + if (type == "stream-topology") + topology = structure.clone(); + + break; + } + + case GST_MESSAGE_ASYNC_DONE: { + if (playerPipeline.state() >= GST_STATE_PAUSED) + detectPipelineIsSeekable(); break; } @@ -447,7 +590,7 @@ bool QGstreamerMediaPlayer::processSyncMessage(const QGstreamerMessage &message) if (message.type() != GST_MESSAGE_NEED_CONTEXT) return false; const gchar *type = nullptr; - gst_message_parse_context_type (message.rawMessage(), &type); + gst_message_parse_context_type (message.message(), &type); if (strcmp(type, GST_GL_DISPLAY_CONTEXT_TYPE)) return false; if (!gstVideoOutput || !gstVideoOutput->gstreamerVideoSink()) @@ -455,7 +598,7 @@ bool QGstreamerMediaPlayer::processSyncMessage(const QGstreamerMessage &message) auto *context = gstVideoOutput->gstreamerVideoSink()->gstGlDisplayContext(); if (!context) return false; - gst_element_set_context(GST_ELEMENT(GST_MESSAGE_SRC(message.rawMessage())), context); + gst_element_set_context(GST_ELEMENT(GST_MESSAGE_SRC(message.message())), context); playerPipeline.dumpGraph("need_context"); return true; #else @@ -482,7 +625,7 @@ void QGstreamerMediaPlayer::decoderPadAdded(const QGstElement &src, const QGstPa auto caps = pad.currentCaps(); auto type = caps.at(0).name(); qCDebug(qLcMediaPlayer) << "Received new pad" << pad.name() << "from" << src.name() << "type" << type; - qCDebug(qLcMediaPlayer) << " " << caps.toString(); + qCDebug(qLcMediaPlayer) << " " << caps; TrackType streamType = NTrackTypes; if (type.startsWith("video/x-raw")) { @@ -508,19 +651,19 @@ void QGstreamerMediaPlayer::decoderPadAdded(const QGstElement &src, const QGstPa if (streamType == VideoStream) { connectOutput(ts); ts.setActiveInputPad(sinkPad); - emit videoAvailableChanged(true); + videoAvailableChanged(true); } else if (streamType == AudioStream) { connectOutput(ts); ts.setActiveInputPad(sinkPad); - emit audioAvailableChanged(true); + audioAvailableChanged(true); } } if (!prerolling) - emit tracksChanged(); + tracksChanged(); - decoderOutputMap.insert(pad.name(), sinkPad); + decoderOutputMap.emplace(pad, sinkPad); } void QGstreamerMediaPlayer::decoderPadRemoved(const QGstElement &src, const QGstPad &pad) @@ -529,9 +672,11 @@ void QGstreamerMediaPlayer::decoderPadRemoved(const QGstElement &src, const QGst return; qCDebug(qLcMediaPlayer) << "Removed pad" << pad.name() << "from" << src.name(); - auto track = decoderOutputMap.value(pad.name()); - if (track.isNull()) + + auto it = decoderOutputMap.find(pad); + if (it == decoderOutputMap.end()) return; + QGstPad track = it->second; auto ts = std::find_if(std::begin(trackSelectors), std::end(trackSelectors), [&](TrackSelector &ts){ return ts.selector == track.parent(); }); @@ -568,27 +713,12 @@ void QGstreamerMediaPlayer::connectOutput(TrackSelector &ts) if (ts.isConnected) return; - QGstElement e; - switch (ts.type) { - case AudioStream: - e = gstAudioOutput ? gstAudioOutput->gstElement() : QGstElement{}; - break; - case VideoStream: - e = gstVideoOutput ? gstVideoOutput->gstElement() : QGstElement{}; - break; - case SubtitleStream: - if (gstVideoOutput) - gstVideoOutput->linkSubtitleStream(ts.selector); - break; - default: - return; - } - - if (!e.isNull()) { + QGstElement e = getSinkElementForTrackType(ts.type); + if (e) { qCDebug(qLcMediaPlayer) << "connecting output for track type" << ts.type; playerPipeline.add(e); - ts.selector.link(e); - e.setState(GST_STATE_PAUSED); + qLinkGstElements(ts.selector, e); + e.syncStateWithParent(); } ts.isConnected = true; @@ -599,47 +729,133 @@ void QGstreamerMediaPlayer::removeOutput(TrackSelector &ts) if (!ts.isConnected) return; - QGstElement e; - switch (ts.type) { - case AudioStream: - e = gstAudioOutput ? gstAudioOutput->gstElement() : QGstElement{}; - break; - case VideoStream: - e = gstVideoOutput ? gstVideoOutput->gstElement() : QGstElement{}; - break; - case SubtitleStream: - if (gstVideoOutput) - gstVideoOutput->unlinkSubtitleStream(); - break; - default: - break; - } - - if (!e.isNull()) { + QGstElement e = getSinkElementForTrackType(ts.type); + if (e) { qCDebug(qLcMediaPlayer) << "removing output for track type" << ts.type; - playerPipeline.remove(e); - e.setStateSync(GST_STATE_NULL); + playerPipeline.stopAndRemoveElements(e); } ts.isConnected = false; } -void QGstreamerMediaPlayer::uridecodebinElementAddedCallback(GstElement */*uridecodebin*/, GstElement *child, QGstreamerMediaPlayer *that) +void QGstreamerMediaPlayer::removeDynamicPipelineElements() { - QGstElement c(child); + for (QGstElement *element : { &src, &decoder }) { + if (element->isNull()) + continue; + + element->setStateSync(GstState::GST_STATE_NULL); + playerPipeline.remove(*element); + *element = QGstElement{}; + } +} + +void QGstreamerMediaPlayer::uridecodebinElementAddedCallback(GstElement * /*uridecodebin*/, + GstElement *child, + QGstreamerMediaPlayer *) +{ + QGstElement c(child, QGstElement::NeedsRef); qCDebug(qLcMediaPlayer) << "New element added to uridecodebin:" << c.name(); - if (G_OBJECT_TYPE(child) == that->decodebinType) { + static const GType decodeBinType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("decodebin"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + if (c.type() == decodeBinType) { qCDebug(qLcMediaPlayer) << " -> setting post-stream-topology property"; c.set("post-stream-topology", true); } } +void QGstreamerMediaPlayer::sourceSetupCallback(GstElement *uridecodebin, GstElement *source, QGstreamerMediaPlayer *that) +{ + Q_UNUSED(uridecodebin) + Q_UNUSED(that) + + qCDebug(qLcMediaPlayer) << "Setting up source:" << g_type_name_from_instance((GTypeInstance*)source); + + if (std::string_view("GstRTSPSrc") == g_type_name_from_instance((GTypeInstance *)source)) { + QGstElement s(source, QGstElement::NeedsRef); + int latency{40}; + bool ok{false}; + int v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_LATENCY", &ok); + if (ok) + latency = v; + qCDebug(qLcMediaPlayer) << " -> setting source latency to:" << latency << "ms"; + s.set("latency", latency); + + bool drop{true}; + v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DROP_ON_LATENCY", &ok); + if (ok && v == 0) + drop = false; + qCDebug(qLcMediaPlayer) << " -> setting drop-on-latency to:" << drop; + s.set("drop-on-latency", drop); + + bool retrans{false}; + v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DO_RETRANSMISSION", &ok); + if (ok && v != 0) + retrans = true; + qCDebug(qLcMediaPlayer) << " -> setting do-retransmission to:" << retrans; + s.set("do-retransmission", retrans); + } +} + +void QGstreamerMediaPlayer::unknownTypeCallback(GstElement *decodebin, GstPad *pad, GstCaps *caps, + QGstreamerMediaPlayer *self) +{ + Q_UNUSED(decodebin) + Q_UNUSED(pad) + Q_UNUSED(self) + qCDebug(qLcMediaPlayer) << "Unknown type:" << caps; + + QMetaObject::invokeMethod(self, [self] { + self->stop(); + }); +} + +static bool isQueue(const QGstElement &element) +{ + static const GType queueType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("queue"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + static const GType multiQueueType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("multiqueue"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + return element.type() == queueType || element.type() == multiQueueType; +} + +void QGstreamerMediaPlayer::decodebinElementAddedCallback(GstBin * /*decodebin*/, + GstBin * /*sub_bin*/, GstElement *child, + QGstreamerMediaPlayer *self) +{ + QGstElement c(child, QGstElement::NeedsRef); + if (isQueue(c)) + self->decodeBinQueues += 1; +} + +void QGstreamerMediaPlayer::decodebinElementRemovedCallback(GstBin * /*decodebin*/, + GstBin * /*sub_bin*/, GstElement *child, + QGstreamerMediaPlayer *self) +{ + QGstElement c(child, QGstElement::NeedsRef); + if (isQueue(c)) + self->decodeBinQueues -= 1; +} + void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) { + using namespace std::chrono_literals; + qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << "setting location to" << content; prerolling = true; + m_requiresSeekOnPlay = true; + m_resourceErrorState = ResourceErrorState::NoError; bool ret = playerPipeline.setStateSync(GST_STATE_NULL); if (!ret) @@ -648,73 +864,104 @@ void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) m_url = content; m_stream = stream; - if (!src.isNull()) - playerPipeline.remove(src); - if (!decoder.isNull()) - playerPipeline.remove(decoder); - src = QGstElement(); - decoder = QGstElement(); + removeDynamicPipelineElements(); + disconnectDecoderHandlers(); removeAllOutputs(); seekableChanged(false); - playerPipeline.setInStoppedState(true); - if (m_duration != 0) { - m_duration = 0; - durationChanged(0); + if (m_duration != 0ms) { + m_duration = 0ms; + durationChanged(0ms); } stateChanged(QMediaPlayer::StoppedState); if (position() != 0) - positionChanged(0); - mediaStatusChanged(QMediaPlayer::NoMedia); + positionChanged(0ms); if (!m_metaData.isEmpty()) { m_metaData.clear(); metaDataChanged(); } - if (content.isEmpty()) + if (content.isEmpty() && !stream) { + mediaStatusChanged(QMediaPlayer::NoMedia); return; + } if (m_stream) { - if (!m_appSrc) - m_appSrc = new QGstAppSrc(this); + if (!m_appSrc) { + auto maybeAppSrc = QGstAppSource::create(this); + if (maybeAppSrc) { + m_appSrc = maybeAppSrc.value(); + } else { + error(QMediaPlayer::ResourceError, maybeAppSrc.error()); + return; + } + } src = m_appSrc->element(); - decoder = QGstElement("decodebin", "decoder"); + decoder = QGstElement::createFromFactory("decodebin", "decoder"); + if (!decoder) { + error(QMediaPlayer::ResourceError, qGstErrorMessageCannotFindElement("decodebin")); + return; + } decoder.set("post-stream-topology", true); + decoder.set("use-buffering", true); + unknownType = decoder.connect("unknown-type", GCallback(unknownTypeCallback), this); + elementAdded = decoder.connect("deep-element-added", + GCallback(decodebinElementAddedCallback), this); + elementRemoved = decoder.connect("deep-element-removed", + GCallback(decodebinElementAddedCallback), this); + playerPipeline.add(src, decoder); - src.link(decoder); + qLinkGstElements(src, decoder); m_appSrc->setup(m_stream); seekableChanged(!stream->isSequential()); } else { // use uridecodebin - decoder = QGstElement("uridecodebin", "uridecoder"); + decoder = QGstElement::createFromFactory("uridecodebin", "decoder"); + if (!decoder) { + error(QMediaPlayer::ResourceError, qGstErrorMessageCannotFindElement("uridecodebin")); + return; + } playerPipeline.add(decoder); - // can't set post-stream-topology to true, as uridecodebin doesn't have the property. Use a hack - decoder.connect("element-added", GCallback(QGstreamerMediaPlayer::uridecodebinElementAddedCallback), this); - decoder.set("uri", content.toEncoded().constData()); - if (m_bufferProgress != 0) { - m_bufferProgress = 0; - emit bufferProgressChanged(0.); + constexpr bool hasPostStreamTopology = GST_CHECK_VERSION(1, 22, 0); + if constexpr (hasPostStreamTopology) { + decoder.set("post-stream-topology", true); + } else { + // can't set post-stream-topology to true, as uridecodebin doesn't have the property. + // Use a hack + uridecodebinElementAdded = decoder.connect( + "element-added", GCallback(uridecodebinElementAddedCallback), this); } + + sourceSetup = decoder.connect("source-setup", GCallback(sourceSetupCallback), this); + unknownType = decoder.connect("unknown-type", GCallback(unknownTypeCallback), this); + + decoder.set("uri", content.toEncoded().constData()); + decoder.set("use-buffering", true); + + constexpr int mb = 1024 * 1024; + decoder.set("ring-buffer-max-size", 2 * mb); + + updateBufferProgress(0.f); + + elementAdded = decoder.connect("deep-element-added", + GCallback(decodebinElementAddedCallback), this); + elementRemoved = decoder.connect("deep-element-removed", + GCallback(decodebinElementAddedCallback), this); } - decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAdded>(this); - decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemoved>(this); + padAdded = decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAdded>(this); + padRemoved = decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemoved>(this); mediaStatusChanged(QMediaPlayer::LoadingMedia); - - if (state() == QMediaPlayer::PlayingState) { - int ret = playerPipeline.setState(GST_STATE_PLAYING); - if (ret == GST_STATE_CHANGE_FAILURE) - qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the playing state."; - } else { - int ret = playerPipeline.setState(GST_STATE_PAUSED); - if (!ret) - qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; + if (!playerPipeline.setStateSync(GST_STATE_PAUSED)) { + qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; + // Note: no further error handling: errors will be delivered via a GstMessage + return; } - playerPipeline.setPosition(0); - positionChanged(0); + playerPipeline.setPosition(0ms); + positionChanged(0ms); } void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) @@ -724,17 +971,14 @@ void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) auto &ts = trackSelector(AudioStream); - playerPipeline.beginConfig(); - if (gstAudioOutput) { - removeOutput(ts); - gstAudioOutput->setPipeline({}); - } - gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); - if (gstAudioOutput) { - gstAudioOutput->setPipeline(playerPipeline); - connectOutput(ts); - } - playerPipeline.endConfig(); + playerPipeline.modifyPipelineWhileNotRunning([&] { + if (gstAudioOutput) + removeOutput(ts); + + gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); + if (gstAudioOutput) + connectOutput(ts); + }); } QMediaMetaData QGstreamerMediaPlayer::metaData() const @@ -747,9 +991,9 @@ void QGstreamerMediaPlayer::setVideoSink(QVideoSink *sink) gstVideoOutput->setVideoSink(sink); } -static QGstStructure endOfChain(const QGstStructure &s) +static QGstStructureView endOfChain(const QGstStructureView &s) { - QGstStructure e = s; + QGstStructureView e = s; while (1) { auto next = e["next"].toStructure(); if (!next.isNull()) @@ -763,31 +1007,26 @@ static QGstStructure endOfChain(const QGstStructure &s) void QGstreamerMediaPlayer::parseStreamsAndMetadata() { qCDebug(qLcMediaPlayer) << "============== parse topology ============"; - if (topology.isNull()) { + + if (!topology) { qCDebug(qLcMediaPlayer) << " null topology"; return; } - auto caps = topology["caps"].toCaps(); - auto structure = caps.at(0); - auto fileFormat = QGstreamerFormatInfo::fileFormatForCaps(structure); - qCDebug(qLcMediaPlayer) << caps.toString() << fileFormat; - m_metaData.insert(QMediaMetaData::FileFormat, QVariant::fromValue(fileFormat)); - m_metaData.insert(QMediaMetaData::Duration, duration()); - m_metaData.insert(QMediaMetaData::Url, m_url); - QGValue tags = topology["tags"]; - if (!tags.isNull()) { - GstTagList *tagList = nullptr; - gst_structure_get(topology.structure, "tags", GST_TYPE_TAG_LIST, &tagList, nullptr); - const auto metaData = QGstreamerMetaData::fromGstTagList(tagList); - for (auto k : metaData.keys()) - m_metaData.insert(k, metaData.value(k)); - } - - auto demux = endOfChain(topology); - auto next = demux["next"]; + + QGstStructureView topologyView{ topology }; + + QGstCaps caps = topologyView.caps(); + extendMetaDataFromCaps(m_metaData, caps); + + QGstTagListHandle tagList = QGstStructureView{ topology }.tags(); + if (tagList) + extendMetaDataFromTagList(m_metaData, tagList); + + QGstStructureView demux = endOfChain(topologyView); + QGValue next = demux["next"]; if (!next.isList()) { qCDebug(qLcMediaPlayer) << " no additional streams"; - emit metaDataChanged(); + metaDataChanged(); return; } @@ -795,38 +1034,28 @@ void QGstreamerMediaPlayer::parseStreamsAndMetadata() int size = next.listSize(); for (int i = 0; i < size; ++i) { auto val = next.at(i); - caps = val.toStructure()["caps"].toCaps(); - structure = caps.at(0); - if (structure.name().startsWith("audio/")) { - auto codec = QGstreamerFormatInfo::audioCodecForCaps(structure); - m_metaData.insert(QMediaMetaData::AudioCodec, QVariant::fromValue(codec)); - qCDebug(qLcMediaPlayer) << " audio" << caps.toString() << (int)codec; - } else if (structure.name().startsWith("video/")) { - auto codec = QGstreamerFormatInfo::videoCodecForCaps(structure); - m_metaData.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(codec)); - qCDebug(qLcMediaPlayer) << " video" << caps.toString() << (int)codec; - auto framerate = structure["framerate"].getFraction(); - if (framerate) - m_metaData.insert(QMediaMetaData::VideoFrameRate, *framerate); - auto width = structure["width"].toInt(); - auto height = structure["height"].toInt(); - if (width && height) - m_metaData.insert(QMediaMetaData::Resolution, QSize(*width, *height)); + caps = val.toStructure().caps(); + + extendMetaDataFromCaps(m_metaData, caps); + + QGstStructureView structure = caps.at(0); + + if (structure.name().startsWith("video/")) { + QSize nativeSize = structure.nativeSize(); + gstVideoOutput->setNativeSize(nativeSize); } } auto sinkPad = trackSelector(VideoStream).activeInputPad(); - if (!sinkPad.isNull()) { - bool hasTags = g_object_class_find_property (G_OBJECT_GET_CLASS (sinkPad.object()), "tags") != NULL; - - GstTagList *tl = nullptr; - g_object_get(sinkPad.object(), "tags", &tl, nullptr); - qCDebug(qLcMediaPlayer) << " tags=" << hasTags << (tl ? gst_tag_list_to_string(tl) : "(null)"); + if (sinkPad) { + QGstTagListHandle tagList = sinkPad.tags(); + if (tagList) + qCDebug(qLcMediaPlayer) << " tags=" << tagList.get(); + else + qCDebug(qLcMediaPlayer) << " tags=(null)"; } - qCDebug(qLcMediaPlayer) << "============== end parse topology ============"; - emit metaDataChanged(); playerPipeline.dumpGraph("playback"); } @@ -838,13 +1067,11 @@ int QGstreamerMediaPlayer::trackCount(QPlatformMediaPlayer::TrackType type) QMediaMetaData QGstreamerMediaPlayer::trackMetaData(QPlatformMediaPlayer::TrackType type, int index) { auto track = trackSelector(type).inputPad(index); - if (track.isNull()) + if (!track) return {}; - GstTagList *tagList = nullptr; - g_object_get(track.object(), "tags", &tagList, nullptr); - - return tagList ? QGstreamerMetaData::fromGstTagList(tagList) : QMediaMetaData{}; + QGstTagListHandle tagList = track.tags(); + return taglistToMetaData(tagList); } int QGstreamerMediaPlayer::activeTrack(TrackType type) @@ -866,14 +1093,14 @@ void QGstreamerMediaPlayer::setActiveTrack(TrackType type, int index) if (type == QPlatformMediaPlayer::SubtitleStream) gstVideoOutput->flushSubtitles(); - playerPipeline.beginConfig(); - if (track.isNull()) { - removeOutput(ts); - } else { - ts.setActiveInputPad(track); - connectOutput(ts); - } - playerPipeline.endConfig(); + playerPipeline.modifyPipelineWhileNotRunning([&] { + if (track.isNull()) { + removeOutput(ts); + } else { + ts.setActiveInputPad(track); + connectOutput(ts); + } + }); // seek to force an immediate change of the stream if (playerPipeline.state() == GST_STATE_PLAYING) diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h index cfc576643..f634d32a1 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMEDIAPLAYER_P_H #define QGSTREAMERMEDIAPLAYER_P_H @@ -54,9 +18,10 @@ #include <QtCore/qstack.h> #include <private/qplatformmediaplayer_p.h> #include <private/qtmultimediaglobal_p.h> +#include <private/qmultimediautils_p.h> #include <qurl.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> #include <QtCore/qtimer.h> @@ -66,23 +31,19 @@ QT_BEGIN_NAMESPACE class QNetworkAccessManager; class QGstreamerMessage; -class QGstAppSrc; +class QGstAppSource; class QGstreamerAudioOutput; class QGstreamerVideoOutput; -class Q_MULTIMEDIA_EXPORT QGstreamerMediaPlayer - : public QObject, - public QPlatformMediaPlayer, - public QGstreamerBusMessageFilter, - public QGstreamerSyncMessageFilter +class QGstreamerMediaPlayer : public QObject, + public QPlatformMediaPlayer, + public QGstreamerBusMessageFilter, + public QGstreamerSyncMessageFilter { - Q_OBJECT - public: - QGstreamerMediaPlayer(QMediaPlayer *parent = 0); + static QMaybe<QPlatformMediaPlayer *> create(QMediaPlayer *parent = nullptr); ~QGstreamerMediaPlayer(); - qint64 position() const override; qint64 duration() const override; float bufferProgress() const override; @@ -94,7 +55,7 @@ public: QUrl media() const override; const QIODevice *mediaStream() const override; - void setMedia(const QUrl&, QIODevice *) override; + void setMedia(const QUrl &, QIODevice *) override; bool streamPlaybackSupported() const override { return true; } @@ -110,26 +71,32 @@ public: void setActiveTrack(TrackType, int /*streamNumber*/) override; void setPosition(qint64 pos) override; + void setPosition(std::chrono::milliseconds pos); void play() override; void pause() override; void stop() override; + const QGstPipeline &pipeline() const; + bool processBusMessage(const QGstreamerMessage& message) override; bool processSyncMessage(const QGstreamerMessage& message) override; -public Q_SLOTS: - void updatePosition() { positionChanged(position()); } - private: - struct TrackSelector { - TrackSelector(TrackType, const char *name); + QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, QMediaPlayer *parent); + + struct TrackSelector + { + TrackSelector(TrackType, QGstElement selector); QGstPad createInputPad(); void removeInputPad(QGstPad pad); void removeAllInputPads(); QGstPad inputPad(int index); int activeInputIndex() const { return isConnected ? tracks.indexOf(activeInputPad()) : -1; } - QGstPad activeInputPad() const { return isConnected ? selector.getObject("active-pad") : QGstPad{}; } + QGstPad activeInputPad() const + { + return isConnected ? QGstPad{ selector.getObject("active-pad") } : QGstPad{}; + } void setActiveInputPad(QGstPad input) { selector.set("active-pad", input); } int trackCount() const { return tracks.count(); } @@ -142,31 +109,61 @@ private: friend class QGstreamerStreamsControl; void decoderPadAdded(const QGstElement &src, const QGstPad &pad); void decoderPadRemoved(const QGstElement &src, const QGstPad &pad); - static void uridecodebinElementAddedCallback(GstElement *uridecodebin, GstElement *child, QGstreamerMediaPlayer *that); + void disconnectDecoderHandlers(); + static void uridecodebinElementAddedCallback(GstElement *uridecodebin, GstElement *child, + QGstreamerMediaPlayer *that); + static void sourceSetupCallback(GstElement *uridecodebin, GstElement *source, + QGstreamerMediaPlayer *that); + static void unknownTypeCallback(GstElement *decodebin, GstPad *pad, GstCaps *caps, + QGstreamerMediaPlayer *self); + static void decodebinElementAddedCallback(GstBin *decodebin, GstBin *sub_bin, + GstElement *element, QGstreamerMediaPlayer *self); + static void decodebinElementRemovedCallback(GstBin *decodebin, GstBin *sub_bin, + GstElement *element, QGstreamerMediaPlayer *self); + void parseStreamsAndMetadata(); void connectOutput(TrackSelector &ts); void removeOutput(TrackSelector &ts); + void removeDynamicPipelineElements(); void removeAllOutputs(); void stopOrEOS(bool eos); + bool canTrackProgress() const { return decodeBinQueues > 0; } + void detectPipelineIsSeekable(); + bool hasMedia() const; + + std::chrono::nanoseconds pipelinePosition() const; + void updatePositionFromPipeline(); + void updateDurationFromPipeline(); + void updateBufferProgress(float); + + QGstElement getSinkElementForTrackType(TrackType); std::array<TrackSelector, NTrackTypes> trackSelectors; TrackSelector &trackSelector(TrackType type); QMediaMetaData m_metaData; - int m_bufferProgress = -1; QUrl m_url; QIODevice *m_stream = nullptr; + enum class ResourceErrorState : uint8_t { + NoError, + ErrorOccurred, + ErrorReported, + }; + bool prerolling = false; - bool m_requiresSeekOnPlay = false; - qint64 m_duration = 0; + bool m_requiresSeekOnPlay = true; + bool m_initialBufferProgressSent = false; + ResourceErrorState m_resourceErrorState = ResourceErrorState::NoError; + float m_rate = 1.f; + float m_bufferProgress = 0.f; + std::chrono::milliseconds m_duration{}; QTimer positionUpdateTimer; - QGstAppSrc *m_appSrc = nullptr; + QGstAppSource *m_appSrc = nullptr; - GType decodebinType; - QGstStructure topology; + QUniqueGstStructureHandle topology; // Gst elements QGstPipeline playerPipeline; @@ -178,7 +175,30 @@ private: // QGstElement streamSynchronizer; - QHash<QByteArray, QGstPad> decoderOutputMap; + struct QGstPadLess + { + bool operator()(const QGstPad &lhs, const QGstPad &rhs) const + { + return lhs.pad() < rhs.pad(); + } + }; + + std::map<QGstPad, QGstPad, QGstPadLess> decoderOutputMap; + + // decoder connections + QGObjectHandlerScopedConnection padAdded; + QGObjectHandlerScopedConnection padRemoved; + QGObjectHandlerScopedConnection sourceSetup; + QGObjectHandlerScopedConnection uridecodebinElementAdded; + QGObjectHandlerScopedConnection unknownType; + QGObjectHandlerScopedConnection elementAdded; + QGObjectHandlerScopedConnection elementRemoved; + + int decodeBinQueues = 0; + + void mediaStatusChanged(QMediaPlayer::MediaStatus status); + static constexpr auto stalledMediaDebouncePeriod = std::chrono::milliseconds{ 500 }; + QTimer m_stalledMediaNotifier; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermessage.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermessage.cpp deleted file mode 100644 index 02a5dd3bc..000000000 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermessage.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <gst/gst.h> - -#include "qgstreamermessage_p.h" - -QT_BEGIN_NAMESPACE - -/*! - \class QGstreamerMessage - \internal -*/ - -QGstreamerMessage::QGstreamerMessage(GstMessage* message): - m_message(message) -{ - gst_message_ref(m_message); -} - -QGstreamerMessage::QGstreamerMessage(QGstreamerMessage const& m): - m_message(m.m_message) -{ - gst_message_ref(m_message); -} - -QGstreamerMessage::QGstreamerMessage(const QGstStructure &structure) -{ - gst_structure_get(structure.structure, "message", GST_TYPE_MESSAGE, &m_message, nullptr); -} - -QGstreamerMessage::~QGstreamerMessage() -{ - if (m_message != nullptr) - gst_message_unref(m_message); -} - -GstMessage* QGstreamerMessage::rawMessage() const -{ - return m_message; -} - -QGstreamerMessage& QGstreamerMessage::operator=(QGstreamerMessage const& rhs) -{ - if (rhs.m_message != m_message) { - if (rhs.m_message != nullptr) - gst_message_ref(rhs.m_message); - - if (m_message != nullptr) - gst_message_unref(m_message); - - m_message = rhs.m_message; - } - - return *this; -} - -QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h index 483f8f5e3..9836bd0cb 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMESSAGE_P_H #define QGSTREAMERMESSAGE_P_H @@ -52,34 +16,36 @@ // #include <private/qtmultimediaglobal_p.h> -#include <qgst_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE // Required for QDoc workaround class QString; -class Q_MULTIMEDIA_EXPORT QGstreamerMessage +template <> +struct QGstPointerImpl::QGstRefcountingAdaptor<GstMessage> { -public: - QGstreamerMessage() = default; - QGstreamerMessage(QGstreamerMessage const& m); - explicit QGstreamerMessage(GstMessage* message); - explicit QGstreamerMessage(const QGstStructure &structure); - - ~QGstreamerMessage(); + static void ref(GstMessage *arg) noexcept { gst_message_ref(arg); } + static void unref(GstMessage *arg) noexcept { gst_message_unref(arg); } +}; - bool isNull() const { return !m_message; } - GstMessageType type() const { return GST_MESSAGE_TYPE(m_message); } - QGstObject source() const { return QGstObject(GST_MESSAGE_SRC(m_message), QGstObject::NeedsRef); } - QGstStructure structure() const { return QGstStructure(gst_message_get_structure(m_message)); } +class QGstreamerMessage : public QGstPointerImpl::QGstObjectWrapper<GstMessage> +{ + using BaseClass = QGstPointerImpl::QGstObjectWrapper<GstMessage>; - GstMessage* rawMessage() const; +public: + using BaseClass::BaseClass; + QGstreamerMessage(const QGstreamerMessage &) = default; + QGstreamerMessage(QGstreamerMessage &&) noexcept = default; + QGstreamerMessage &operator=(const QGstreamerMessage &) = default; + QGstreamerMessage &operator=(QGstreamerMessage &&) noexcept = default; - QGstreamerMessage& operator=(QGstreamerMessage const& rhs); + GstMessageType type() const { return GST_MESSAGE_TYPE(get()); } + QGstObject source() const { return QGstObject(GST_MESSAGE_SRC(get()), QGstObject::NeedsRef); } + QGstStructureView structure() const { return QGstStructureView(gst_message_get_structure(get())); } -private: - GstMessage* m_message = nullptr; + GstMessage *message() const { return get(); } }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp index 55f5ef155..9aa9406b9 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp @@ -1,308 +1,489 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamermetadata_p.h" -#include <QDebug> #include <QtMultimedia/qmediametadata.h> +#include <QtMultimedia/qtvideo.h> +#include <QtCore/qdebug.h> #include <QtCore/qdatetime.h> +#include <QtCore/qlocale.h> +#include <QtCore/qtimezone.h> +#include <QtGui/qimage.h> #include <gst/gstversion.h> -#include <qgstutils_p.h> -#include <qlocale.h> +#include <common/qgst_handle_types_p.h> +#include <common/qgstutils_p.h> +#include <qgstreamerformatinfo_p.h> QT_BEGIN_NAMESPACE -struct { +namespace { + +namespace MetadataLookupImpl { + +#ifdef __cpp_lib_constexpr_algorithms +# define constexpr_lookup constexpr +#else +# define constexpr_lookup /*constexpr*/ +#endif + +struct MetadataKeyValuePair +{ const char *tag; QMediaMetaData::Key key; -} gstTagToMetaDataKey[] = { - { GST_TAG_TITLE, QMediaMetaData::Title }, - { GST_TAG_COMMENT, QMediaMetaData::Comment }, - { GST_TAG_DESCRIPTION, QMediaMetaData::Description }, - { GST_TAG_GENRE, QMediaMetaData::Genre }, - { GST_TAG_DATE_TIME, QMediaMetaData::Date }, - { GST_TAG_DATE, QMediaMetaData::Date }, +}; + +constexpr const char *toTag(const char *t) +{ + return t; +} +constexpr const char *toTag(const MetadataKeyValuePair &kv) +{ + return kv.tag; +} + +constexpr QMediaMetaData::Key toKey(QMediaMetaData::Key k) +{ + return k; +} +constexpr QMediaMetaData::Key toKey(const MetadataKeyValuePair &kv) +{ + return kv.key; +} + +constexpr auto compareByKey = [](const auto &lhs, const auto &rhs) { + return toKey(lhs) < toKey(rhs); +}; - { GST_TAG_LANGUAGE_CODE, QMediaMetaData::Language }, +constexpr auto compareByTag = [](const auto &lhs, const auto &rhs) { + return std::strcmp(toTag(lhs), toTag(rhs)) < 0; +}; + +constexpr_lookup auto makeLookupTable() +{ + std::array<MetadataKeyValuePair, 22> lookupTable{ { + { GST_TAG_TITLE, QMediaMetaData::Title }, + { GST_TAG_COMMENT, QMediaMetaData::Comment }, + { GST_TAG_DESCRIPTION, QMediaMetaData::Description }, + { GST_TAG_GENRE, QMediaMetaData::Genre }, + { GST_TAG_DATE_TIME, QMediaMetaData::Date }, + { GST_TAG_DATE, QMediaMetaData::Date }, + + { GST_TAG_LANGUAGE_CODE, QMediaMetaData::Language }, + + { GST_TAG_ORGANIZATION, QMediaMetaData::Publisher }, + { GST_TAG_COPYRIGHT, QMediaMetaData::Copyright }, + + // Media + { GST_TAG_DURATION, QMediaMetaData::Duration }, + + // Audio + { GST_TAG_BITRATE, QMediaMetaData::AudioBitRate }, + { GST_TAG_AUDIO_CODEC, QMediaMetaData::AudioCodec }, + + // Music + { GST_TAG_ALBUM, QMediaMetaData::AlbumTitle }, + { GST_TAG_ALBUM_ARTIST, QMediaMetaData::AlbumArtist }, + { GST_TAG_ARTIST, QMediaMetaData::ContributingArtist }, + { GST_TAG_TRACK_NUMBER, QMediaMetaData::TrackNumber }, + + { GST_TAG_PREVIEW_IMAGE, QMediaMetaData::ThumbnailImage }, + { GST_TAG_IMAGE, QMediaMetaData::CoverArtImage }, + + // Image/Video + { "resolution", QMediaMetaData::Resolution }, + { GST_TAG_IMAGE_ORIENTATION, QMediaMetaData::Orientation }, + + // Video + { GST_TAG_VIDEO_CODEC, QMediaMetaData::VideoCodec }, + + // Movie + { GST_TAG_PERFORMER, QMediaMetaData::LeadPerformer }, + } }; + + std::sort(lookupTable.begin(), lookupTable.end(), + [](const MetadataKeyValuePair &lhs, const MetadataKeyValuePair &rhs) { + return std::string_view(lhs.tag) < std::string_view(rhs.tag); + }); + return lookupTable; +} - { GST_TAG_ORGANIZATION, QMediaMetaData::Publisher }, - { GST_TAG_COPYRIGHT, QMediaMetaData::Copyright }, +constexpr_lookup auto gstTagToMetaDataKey = makeLookupTable(); +constexpr_lookup auto metaDataKeyToGstTag = [] { + auto array = gstTagToMetaDataKey; + std::sort(array.begin(), array.end(), compareByKey); + return array; +}(); - // Media - { GST_TAG_DURATION, QMediaMetaData::Duration }, +} // namespace MetadataLookupImpl - // Audio - { GST_TAG_BITRATE, QMediaMetaData::AudioBitRate }, - { GST_TAG_AUDIO_CODEC, QMediaMetaData::AudioCodec }, +QMediaMetaData::Key tagToKey(const char *tag) +{ + if (tag == nullptr) + return QMediaMetaData::Key(-1); - // Music - { GST_TAG_ALBUM, QMediaMetaData::AlbumTitle }, - { GST_TAG_ALBUM_ARTIST, QMediaMetaData::AlbumArtist }, - { GST_TAG_ARTIST, QMediaMetaData::ContributingArtist }, - { GST_TAG_TRACK_NUMBER, QMediaMetaData::TrackNumber }, + using namespace MetadataLookupImpl; + auto foundIterator = std::lower_bound(gstTagToMetaDataKey.begin(), gstTagToMetaDataKey.end(), + tag, compareByTag); + if (std::strcmp(foundIterator->tag, tag) == 0) + return foundIterator->key; - { GST_TAG_PREVIEW_IMAGE, QMediaMetaData::ThumbnailImage }, - { GST_TAG_IMAGE, QMediaMetaData::CoverArtImage }, + return QMediaMetaData::Key(-1); +} - // Image/Video - { "resolution", QMediaMetaData::Resolution }, - { GST_TAG_IMAGE_ORIENTATION, QMediaMetaData::Orientation }, +const char *keyToTag(QMediaMetaData::Key key) +{ + using namespace MetadataLookupImpl; + auto foundIterator = std::lower_bound(metaDataKeyToGstTag.begin(), metaDataKeyToGstTag.end(), + key, compareByKey); + if (foundIterator->key == key) + return foundIterator->tag; - // Video - { GST_TAG_VIDEO_CODEC, QMediaMetaData::VideoCodec }, + return nullptr; +} - // Movie - { GST_TAG_PERFORMER, QMediaMetaData::LeadPerformer }, +#undef constexpr_lookup - { nullptr, QMediaMetaData::Title } -}; +QtVideo::Rotation parseRotationTag(const char *string) +{ + using namespace std::string_view_literals; + + if (string == "rotate-90"sv) + return QtVideo::Rotation::Clockwise90; + if (string == "rotate-180"sv) + return QtVideo::Rotation::Clockwise180; + if (string == "rotate-270"sv) + return QtVideo::Rotation::Clockwise270; + if (string == "rotate-0"sv) + return QtVideo::Rotation::None; + + qCritical() << "cannot parse orientation: {}" << string; + return QtVideo::Rotation::None; +} + +QDateTime parseDate(const GValue &val) +{ + Q_ASSERT(G_VALUE_TYPE(&val) == G_TYPE_DATE); + + const GDate *date = (const GDate *)g_value_get_boxed(&val); + if (!g_date_valid(date)) + return {}; + + int year = g_date_get_year(date); + int month = g_date_get_month(date); + int day = g_date_get_day(date); + return QDateTime(QDate(year, month, day), QTime()); +} -static QMediaMetaData::Key tagToKey(const char *tag) +QDateTime parseDateTime(const GValue &val) { - auto *map = gstTagToMetaDataKey; - while (map->tag) { - if (!strcmp(map->tag, tag)) - return map->key; - ++map; + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_DATE_TIME); + + const GstDateTime *dateTime = (const GstDateTime *)g_value_get_boxed(&val); + int year = gst_date_time_has_year(dateTime) ? gst_date_time_get_year(dateTime) : 0; + int month = gst_date_time_has_month(dateTime) ? gst_date_time_get_month(dateTime) : 0; + int day = gst_date_time_has_day(dateTime) ? gst_date_time_get_day(dateTime) : 0; + int hour = 0; + int minute = 0; + int second = 0; + float tz = 0; + if (gst_date_time_has_time(dateTime)) { + hour = gst_date_time_get_hour(dateTime); + minute = gst_date_time_get_minute(dateTime); + second = gst_date_time_get_second(dateTime); + tz = gst_date_time_get_time_zone_offset(dateTime); } - return QMediaMetaData::Key(-1); + return QDateTime{ + QDate(year, month, day), + QTime(hour, minute, second), + QTimeZone(tz * 60 * 60), + }; } -static const char *keyToTag(QMediaMetaData::Key key) +QImage parseImage(const GValue &val) { - auto *map = gstTagToMetaDataKey; - while (map->tag) { - if (map->key == key) - return map->tag; - ++map; + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_SAMPLE); + + GstSample *sample = (GstSample *)g_value_get_boxed(&val); + GstCaps *caps = gst_sample_get_caps(sample); + if (caps && !gst_caps_is_empty(caps)) { + GstStructure *structure = gst_caps_get_structure(caps, 0); + const gchar *name = gst_structure_get_name(structure); + if (QByteArray(name).startsWith("image/")) { + GstBuffer *buffer = gst_sample_get_buffer(sample); + if (buffer) { + GstMapInfo info; + gst_buffer_map(buffer, &info, GST_MAP_READ); + QImage image = QImage::fromData(info.data, info.size, name); + gst_buffer_unmap(buffer, &info); + return image; + } + } } - return nullptr; + + return {}; +} + +std::optional<double> parseFractionAsDouble(const GValue &val) +{ + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_FRACTION); + + int nom = gst_value_get_fraction_numerator(&val); + int denom = gst_value_get_fraction_denominator(&val); + if (denom == 0) + return std::nullopt; + return double(nom) / double(denom); } -//internal -static void addTagToMap(const GstTagList *list, - const gchar *tag, - gpointer user_data) +constexpr std::string_view extendedComment{ GST_TAG_EXTENDED_COMMENT }; + +void addTagsFromExtendedComment(const GstTagList *list, const gchar *tag, QMediaMetaData &metadata) { + using namespace Qt::Literals; + assert(tag == extendedComment); + + int entryCount = gst_tag_list_get_tag_size(list, tag); + for (int i = 0; i != entryCount; ++i) { + const GValue *value = gst_tag_list_get_value_index(list, tag, i); + + const QLatin1StringView strValue{ g_value_get_string(value) }; + + auto equalIndex = strValue.indexOf(QLatin1StringView("=")); + if (equalIndex == -1) { + qDebug() << "Cannot parse GST_TAG_EXTENDED_COMMENT entry: " << value; + continue; + } + + const QLatin1StringView key = strValue.first(equalIndex); + const QLatin1StringView valueString = strValue.last(strValue.size() - equalIndex - 1); + + if (key == "DURATION"_L1) { + QUniqueGstDateTimeHandle duration{ + gst_date_time_new_from_iso8601_string(valueString.data()), + }; + + if (duration) { + using namespace std::chrono; + + auto chronoDuration = hours(gst_date_time_get_hour(duration.get())) + + minutes(gst_date_time_get_minute(duration.get())) + + seconds(gst_date_time_get_second(duration.get())) + + microseconds(gst_date_time_get_microsecond(duration.get())); + + metadata.insert(QMediaMetaData::Duration, + QVariant::fromValue(round<milliseconds>(chronoDuration).count())); + } + } + } +} + +void addTagToMetaData(const GstTagList *list, const gchar *tag, void *userdata) +{ + QMediaMetaData &metadata = *reinterpret_cast<QMediaMetaData *>(userdata); + QMediaMetaData::Key key = tagToKey(tag); - if (key == QMediaMetaData::Key(-1)) - return; + if (key == QMediaMetaData::Key(-1)) { + if (tag == extendedComment) + addTagsFromExtendedComment(list, tag, metadata); - auto *map = reinterpret_cast<QHash<QMediaMetaData::Key, QVariant>* >(user_data); + return; + } - GValue val; - val.g_type = 0; + GValue val{}; gst_tag_list_copy_value(&val, list, tag); + GType type = G_VALUE_TYPE(&val); - switch( G_VALUE_TYPE(&val) ) { - case G_TYPE_STRING: - { - const gchar *str_value = g_value_get_string(&val); - if (key == QMediaMetaData::Language) { - map->insert(key, QVariant::fromValue(QLocale::codeToLanguage(QString::fromUtf8(str_value), QLocale::ISO639Part2))); - break; - } - map->insert(key, QString::fromUtf8(str_value)); + if (auto entryCount = gst_tag_list_get_tag_size(list, tag) != 0; entryCount != 1) + qWarning() << "addTagToMetaData: invaled entry count for" << tag << "-" << entryCount; + + if (type == G_TYPE_STRING) { + const gchar *str_value = g_value_get_string(&val); + + switch (key) { + case QMediaMetaData::Language: { + metadata.insert(key, + QVariant::fromValue(QLocale::codeToLanguage( + QString::fromUtf8(str_value), QLocale::AnyLanguageCode))); break; } - case G_TYPE_INT: - map->insert(key, g_value_get_int(&val)); - break; - case G_TYPE_UINT: - map->insert(key, g_value_get_uint(&val)); - break; - case G_TYPE_LONG: - map->insert(key, qint64(g_value_get_long(&val))); - break; - case G_TYPE_BOOLEAN: - map->insert(key, g_value_get_boolean(&val)); - break; - case G_TYPE_CHAR: - map->insert(key, g_value_get_schar(&val)); - break; - case G_TYPE_DOUBLE: - map->insert(key, g_value_get_double(&val)); + case QMediaMetaData::Orientation: { + metadata.insert(key, QVariant::fromValue(parseRotationTag(str_value))); break; + } default: - // GST_TYPE_DATE is a function, not a constant, so pull it out of the switch - if (G_VALUE_TYPE(&val) == G_TYPE_DATE) { - const GDate *date = (const GDate *)g_value_get_boxed(&val); - if (g_date_valid(date)) { - int year = g_date_get_year(date); - int month = g_date_get_month(date); - int day = g_date_get_day(date); - // don't insert if we already have a datetime. - if (!map->contains(key)) - map->insert(key, QDateTime(QDate(year, month, day), QTime())); - } - } else if (G_VALUE_TYPE(&val) == GST_TYPE_DATE_TIME) { - const GstDateTime *dateTime = (const GstDateTime *)g_value_get_boxed(&val); - int year = gst_date_time_has_year(dateTime) ? gst_date_time_get_year(dateTime) : 0; - int month = gst_date_time_has_month(dateTime) ? gst_date_time_get_month(dateTime) : 0; - int day = gst_date_time_has_day(dateTime) ? gst_date_time_get_day(dateTime) : 0; - int hour = 0; - int minute = 0; - int second = 0; - float tz = 0; - if (gst_date_time_has_time(dateTime)) { - hour = gst_date_time_get_hour(dateTime); - minute = gst_date_time_get_minute(dateTime); - second = gst_date_time_get_second(dateTime); - tz = gst_date_time_get_time_zone_offset(dateTime); - } - QDateTime qDateTime(QDate(year, month, day), QTime(hour, minute, second), - Qt::OffsetFromUTC, tz * 60 * 60); - map->insert(key, qDateTime); - } else if (G_VALUE_TYPE(&val) == GST_TYPE_SAMPLE) { - GstSample *sample = (GstSample *)g_value_get_boxed(&val); - GstCaps* caps = gst_sample_get_caps(sample); - if (caps && !gst_caps_is_empty(caps)) { - GstStructure *structure = gst_caps_get_structure(caps, 0); - const gchar *name = gst_structure_get_name(structure); - if (QByteArray(name).startsWith("image/")) { - GstBuffer *buffer = gst_sample_get_buffer(sample); - if (buffer) { - GstMapInfo info; - gst_buffer_map(buffer, &info, GST_MAP_READ); - map->insert(key, QImage::fromData(info.data, info.size, name)); - gst_buffer_unmap(buffer, &info); - } - } - } - } else if (G_VALUE_TYPE(&val) == GST_TYPE_FRACTION) { - int nom = gst_value_get_fraction_numerator(&val); - int denom = gst_value_get_fraction_denominator(&val); - - if (denom > 0) { - map->insert(key, double(nom)/denom); - } - } + metadata.insert(key, QString::fromUtf8(str_value)); break; + }; + } else if (type == G_TYPE_INT) { + metadata.insert(key, g_value_get_int(&val)); + } else if (type == G_TYPE_UINT) { + metadata.insert(key, g_value_get_uint(&val)); + } else if (type == G_TYPE_LONG) { + metadata.insert(key, qint64(g_value_get_long(&val))); + } else if (type == G_TYPE_BOOLEAN) { + metadata.insert(key, g_value_get_boolean(&val)); + } else if (type == G_TYPE_CHAR) { + metadata.insert(key, g_value_get_schar(&val)); + } else if (type == G_TYPE_DOUBLE) { + metadata.insert(key, g_value_get_double(&val)); + } else if (type == G_TYPE_DATE) { + if (!metadata.keys().contains(key)) { + QDateTime date = parseDate(val); + if (date.isValid()) + metadata.insert(key, date); + } + } else if (type == GST_TYPE_DATE_TIME) { + metadata.insert(key, parseDateTime(val)); + } else if (type == GST_TYPE_SAMPLE) { + QImage image = parseImage(val); + if (!image.isNull()) + metadata.insert(key, image); + } else if (type == GST_TYPE_FRACTION) { + std::optional<double> fraction = parseFractionAsDouble(val); + + if (fraction) + metadata.insert(key, *fraction); } g_value_unset(&val); } +} // namespace -QGstreamerMetaData QGstreamerMetaData::fromGstTagList(const GstTagList *tags) +QMediaMetaData taglistToMetaData(const QGstTagListHandle &handle) { - QGstreamerMetaData m; - gst_tag_list_foreach(tags, addTagToMap, &m.data); + QMediaMetaData m; + extendMetaDataFromTagList(m, handle); return m; } - -void QGstreamerMetaData::setMetaData(GstElement *element) const +void extendMetaDataFromTagList(QMediaMetaData &metadata, const QGstTagListHandle &handle) { - if (!GST_IS_TAG_SETTER(element)) - return; + if (handle) + gst_tag_list_foreach(handle.get(), reinterpret_cast<GstTagForeachFunc>(&addTagToMetaData), + &metadata); +} - gst_tag_setter_reset_tags(GST_TAG_SETTER(element)); +static void applyMetaDataToTagSetter(const QMediaMetaData &metadata, GstTagSetter *element) +{ + gst_tag_setter_reset_tags(element); - for (auto it = data.cbegin(), end = data.cend(); it != end; ++it) { - const char *tagName = keyToTag(it.key()); + for (QMediaMetaData::Key key : metadata.keys()) { + const char *tagName = keyToTag(key); if (!tagName) continue; - const QVariant &tagValue = it.value(); + const QVariant &tagValue = metadata.value(key); + + auto setTag = [&](const auto &value) { + gst_tag_setter_add_tags(element, GST_TAG_MERGE_REPLACE, tagName, value, nullptr); + }; switch (tagValue.typeId()) { - case QMetaType::QString: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toString().toUtf8().constData(), - nullptr); - break; - case QMetaType::Int: - case QMetaType::LongLong: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toInt(), - nullptr); - break; - case QMetaType::Double: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toDouble(), - nullptr); - break; - case QMetaType::QDate: - case QMetaType::QDateTime: { - QDateTime date = tagValue.toDateTime(); - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - gst_date_time_new(date.offsetFromUtc() / 60. / 60., - date.date().year(), date.date().month(), date.date().day(), - date.time().hour(), date.time().minute(), date.time().second()), - nullptr); - break; - } - default: { - if (tagValue.typeId() == qMetaTypeId<QLocale::Language>()) { - QByteArray language = QLocale::languageToCode(tagValue.value<QLocale::Language>(), QLocale::ISO639Part2).toUtf8(); - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - language.constData(), - nullptr); - } - - break; + case QMetaType::QString: + setTag(tagValue.toString().toUtf8().constData()); + break; + case QMetaType::Int: + case QMetaType::LongLong: + setTag(tagValue.toInt()); + break; + case QMetaType::Double: + setTag(tagValue.toDouble()); + break; + case QMetaType::QDate: + case QMetaType::QDateTime: { + QDateTime date = tagValue.toDateTime(); + + QGstGstDateTimeHandle dateTime{ + gst_date_time_new(date.offsetFromUtc() / 60. / 60., date.date().year(), + date.date().month(), date.date().day(), date.time().hour(), + date.time().minute(), date.time().second()), + QGstGstDateTimeHandle::HasRef, + }; + + setTag(dateTime.get()); + break; + } + default: { + if (tagValue.typeId() == qMetaTypeId<QLocale::Language>()) { + QByteArray language = QLocale::languageToCode(tagValue.value<QLocale::Language>(), + QLocale::ISO639Part2) + .toUtf8(); + setTag(language.constData()); } + + break; + } } } } -void QGstreamerMetaData::setMetaData(GstBin *bin) const +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstElement &element) +{ + GstTagSetter *tagSetter = qGstSafeCast<GstTagSetter>(element.element()); + if (tagSetter) + applyMetaDataToTagSetter(metadata, tagSetter); + else + qWarning() << "applyMetaDataToTagSetter failed: element not a GstTagSetter" + << element.name(); +} + +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstBin &bin) { - GstIterator *elements = gst_bin_iterate_all_by_interface(bin, GST_TYPE_TAG_SETTER); - GValue item = G_VALUE_INIT; + GstIterator *elements = gst_bin_iterate_all_by_interface(bin.bin(), GST_TYPE_TAG_SETTER); + GValue item = {}; + while (gst_iterator_next(elements, &item) == GST_ITERATOR_OK) { - GstElement * const element = GST_ELEMENT(g_value_get_object(&item)); - setMetaData(element); + GstElement *element = static_cast<GstElement *>(g_value_get_object(&item)); + if (!element) + continue; + + GstTagSetter *tagSetter = qGstSafeCast<GstTagSetter>(element); + + if (tagSetter) + applyMetaDataToTagSetter(metadata, tagSetter); } + gst_iterator_free(elements); } +void extendMetaDataFromCaps(QMediaMetaData &metadata, const QGstCaps &caps) +{ + QGstStructureView structure = caps.at(0); + + QMediaFormat::FileFormat fileFormat = QGstreamerFormatInfo::fileFormatForCaps(structure); + if (fileFormat != QMediaFormat::FileFormat::UnspecifiedFormat) { + // Container caps + metadata.insert(QMediaMetaData::FileFormat, fileFormat); + return; + } + + QMediaFormat::AudioCodec audioCodec = QGstreamerFormatInfo::audioCodecForCaps(structure); + if (audioCodec != QMediaFormat::AudioCodec::Unspecified) { + // Audio stream caps + metadata.insert(QMediaMetaData::AudioCodec, QVariant::fromValue(audioCodec)); + return; + } + + QMediaFormat::VideoCodec videoCodec = QGstreamerFormatInfo::videoCodecForCaps(structure); + if (videoCodec != QMediaFormat::VideoCodec::Unspecified) { + // Video stream caps + metadata.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(videoCodec)); + std::optional<float> framerate = structure["framerate"].getFraction(); + if (framerate) + metadata.insert(QMediaMetaData::VideoFrameRate, *framerate); + + QSize resolution = structure.resolution(); + if (resolution.isValid()) + metadata.insert(QMediaMetaData::Resolution, resolution); + } +} + +QMediaMetaData capsToMetaData(const QGstCaps &caps) +{ + QMediaMetaData metadata; + extendMetaDataFromCaps(metadata, caps); + return metadata; +} QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h index 53b29f548..f04a9aba9 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMETADATA_H #define QGSTREAMERMETADATA_H @@ -52,21 +16,19 @@ // #include <qmediametadata.h> -#include <qvariant.h> -#include <gst/gst.h> +#include "qgst_p.h" QT_BEGIN_NAMESPACE -class QGstreamerMetaData : public QMediaMetaData -{ -public: - static QGstreamerMetaData fromGstTagList(const GstTagList *tags); - GstTagList *toGstTagList() const; +QMediaMetaData taglistToMetaData(const QGstTagListHandle &); +void extendMetaDataFromTagList(QMediaMetaData &, const QGstTagListHandle &); - void setMetaData(GstBin *bin) const; - void setMetaData(GstElement *element) const; -}; +QMediaMetaData capsToMetaData(const QGstCaps &); +void extendMetaDataFromCaps(QMediaMetaData &, const QGstCaps &); + +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstBin &); +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstElement &); QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp index d064f593e..40ff5cd85 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp @@ -1,194 +1,174 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreamervideooutput_p.h> -#include <qgstreamervideosink_p.h> -#include <qgstsubtitlesink_p.h> -#include <qvideosink.h> +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/qvideosink.h> #include <QtCore/qloggingcategory.h> -#include <qthread.h> +#include <QtCore/qthread.h> + +#include <common/qgstreamervideooutput_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgstsubtitlesink_p.h> -Q_LOGGING_CATEGORY(qLcMediaVideoOutput, "qt.multimedia.videooutput") +static Q_LOGGING_CATEGORY(qLcMediaVideoOutput, "qt.multimedia.videooutput") QT_BEGIN_NAMESPACE +static QGstElement makeVideoConvertScale(const char *name) +{ + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + if (factory) // videoconvertscale is only available in gstreamer 1.20 + return QGstElement::createFromFactory(factory, name); + + return QGstBin::createFromPipelineDescription("videoconvert ! videoscale", name, + /*ghostUnlinkedPads=*/true); +} + +QMaybe<QGstreamerVideoOutput *> QGstreamerVideoOutput::create(QObject *parent) +{ + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + + static std::optional<QString> elementCheck = []() -> std::optional<QString> { + std::optional<QString> error = qGstErrorMessageIfElementsNotAvailable("fakesink", "queue"); + if (error) + return error; + + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + if (factory) + return std::nullopt; + + return qGstErrorMessageIfElementsNotAvailable("videoconvert", "videoscale"); + }(); + + if (elementCheck) + return *elementCheck; + + return new QGstreamerVideoOutput(parent); +} + QGstreamerVideoOutput::QGstreamerVideoOutput(QObject *parent) : QObject(parent), - gstVideoOutput("videoOutput") + m_outputBin{ + QGstBin::create("videoOutput"), + }, + m_videoQueue{ + QGstElement::createFromFactory("queue", "videoQueue"), + }, + m_videoConvertScale{ + makeVideoConvertScale("videoConvertScale"), + }, + m_videoSink{ + QGstElement::createFromFactory("fakesink", "fakeVideoSink"), + } { - videoQueue = QGstElement("queue", "videoQueue"); - videoConvert = QGstElement("videoconvert", "videoConvert"); - videoSink = QGstElement("fakesink", "fakeVideoSink"); - videoSink.set("sync", true); - gstVideoOutput.add(videoQueue, videoConvert, videoSink); - if (!videoQueue.link(videoConvert, videoSink)) - qCDebug(qLcMediaVideoOutput) << ">>>>>> linking failed"; - - gstVideoOutput.addGhostPad(videoQueue, "sink"); + m_videoSink.set("sync", true); + m_videoSink.set("async", false); // no asynchronous state changes + + m_outputBin.add(m_videoQueue, m_videoConvertScale, m_videoSink); + qLinkGstElements(m_videoQueue, m_videoConvertScale, m_videoSink); + + m_subtitleSink = QGstSubtitleSink::createSink(this); + + m_outputBin.addGhostPad(m_videoQueue, "sink"); } QGstreamerVideoOutput::~QGstreamerVideoOutput() { - gstVideoOutput.setStateSync(GST_STATE_NULL); + QObject::disconnect(m_subtitleConnection); + m_outputBin.setStateSync(GST_STATE_NULL); } void QGstreamerVideoOutput::setVideoSink(QVideoSink *sink) { auto *gstVideoSink = sink ? static_cast<QGstreamerVideoSink *>(sink->platformVideoSink()) : nullptr; - if (gstVideoSink == m_videoSink) + if (gstVideoSink == m_platformVideoSink) return; - if (m_videoSink) - m_videoSink->setPipeline({}); - - m_videoSink = gstVideoSink; - if (m_videoSink) - m_videoSink->setPipeline(gstPipeline); - - QGstElement gstSink; - if (m_videoSink) { - gstSink = m_videoSink->gstSink(); - isFakeSink = false; + m_platformVideoSink = gstVideoSink; + if (m_platformVideoSink) { + m_platformVideoSink->setActive(m_isActive); + if (m_nativeSize.isValid()) + m_platformVideoSink->setNativeSize(m_nativeSize); + } + QGstElement videoSink; + if (m_platformVideoSink) { + videoSink = m_platformVideoSink->gstSink(); } else { - gstSink = QGstElement("fakesink", "fakevideosink"); - gstSink.set("sync", true); - isFakeSink = true; + videoSink = QGstElement::createFromFactory("fakesink", "fakevideosink"); + Q_ASSERT(videoSink); + videoSink.set("sync", true); + videoSink.set("async", false); // no asynchronous state changes } - if (videoSink == gstSink) - return; - - gstPipeline.beginConfig(); - if (!videoSink.isNull()) { - gstVideoOutput.remove(videoSink); - videoSink.setStateSync(GST_STATE_NULL); + QObject::disconnect(m_subtitleConnection); + if (sink) { + m_subtitleConnection = QObject::connect(this, &QGstreamerVideoOutput::subtitleChanged, sink, + [sink](const QString &subtitle) { + sink->setSubtitleText(subtitle); + }); + sink->setSubtitleText(m_lastSubtitleString); } - videoSink = gstSink; - gstVideoOutput.add(videoSink); - videoConvert.link(videoSink); - GstEvent *event = gst_event_new_reconfigure(); - gst_element_send_event(videoSink.element(), event); - videoSink.syncStateWithParent(); + if (m_videoSink == videoSink) + return; + + m_pipeline.modifyPipelineWhileNotRunning([&] { + if (!m_videoSink.isNull()) + m_outputBin.stopAndRemoveElements(m_videoSink); - doLinkSubtitleStream(); + m_videoSink = videoSink; + m_outputBin.add(m_videoSink); - gstPipeline.endConfig(); + qLinkGstElements(m_videoConvertScale, m_videoSink); - qCDebug(qLcMediaVideoOutput) << "sinkChanged" << gstSink.name(); + GstEvent *event = gst_event_new_reconfigure(); + gst_element_send_event(m_videoSink.element(), event); + m_videoSink.syncStateWithParent(); + }); - GST_DEBUG_BIN_TO_DOT_FILE(gstPipeline.bin(), - GstDebugGraphDetails(/*GST_DEBUG_GRAPH_SHOW_ALL |*/ GST_DEBUG_GRAPH_SHOW_MEDIA_TYPE | - GST_DEBUG_GRAPH_SHOW_NON_DEFAULT_PARAMS | GST_DEBUG_GRAPH_SHOW_STATES), - videoSink.name()); + qCDebug(qLcMediaVideoOutput) << "sinkChanged" << videoSink.name(); + m_pipeline.dumpGraph(m_videoSink.name().constData()); } void QGstreamerVideoOutput::setPipeline(const QGstPipeline &pipeline) { - gstPipeline = pipeline; - if (m_videoSink) - m_videoSink->setPipeline(gstPipeline); + m_pipeline = pipeline; } -void QGstreamerVideoOutput::linkSubtitleStream(QGstElement src) +void QGstreamerVideoOutput::setActive(bool isActive) { - qCDebug(qLcMediaVideoOutput) << "link subtitle stream" << src.isNull(); - if (src == subtitleSrc) + if (m_isActive == isActive) return; - gstPipeline.beginConfig(); - subtitleSrc = src; - doLinkSubtitleStream(); - gstPipeline.endConfig(); + m_isActive = isActive; + if (m_platformVideoSink) + m_platformVideoSink->setActive(isActive); } -void QGstreamerVideoOutput::unlinkSubtitleStream() +void QGstreamerVideoOutput::updateNativeSize() { - if (subtitleSrc.isNull()) + if (!m_platformVideoSink) return; - qCDebug(qLcMediaVideoOutput) << "unlink subtitle stream"; - subtitleSrc = {}; - if (!subtitleSink.isNull()) { - gstPipeline.beginConfig(); - gstPipeline.remove(subtitleSink); - gstPipeline.endConfig(); - subtitleSink.setStateSync(GST_STATE_NULL); - subtitleSink = {}; - } - if (m_videoSink) - m_videoSink->setSubtitleText({}); -} -void QGstreamerVideoOutput::doLinkSubtitleStream() -{ - if (!subtitleSink.isNull()) { - gstPipeline.remove(subtitleSink); - subtitleSink.setStateSync(GST_STATE_NULL); - subtitleSink = {}; - } - if (!m_videoSink || subtitleSrc.isNull()) - return; - if (subtitleSink.isNull()) { - subtitleSink = m_videoSink->subtitleSink(); - gstPipeline.add(subtitleSink); - } - if (!subtitleSrc.link(subtitleSink)) - qCDebug(qLcMediaVideoOutput) << "link subtitle stream failed"; + m_platformVideoSink->setNativeSize(qRotatedFrameSize(m_nativeSize, m_rotation)); } void QGstreamerVideoOutput::setIsPreview() { // configures the queue to be fast and lightweight for camera preview // also avoids blocking the queue in case we have an encodebin attached to the tee as well - videoQueue.set("leaky", 2 /*downstream*/); - videoQueue.set("silent", true); - videoQueue.set("max-size-buffers", uint(1)); - videoQueue.set("max-size-bytes", uint(0)); - videoQueue.set("max-size-time", quint64(0)); + m_videoQueue.set("leaky", 2 /*downstream*/); + m_videoQueue.set("silent", true); + m_videoQueue.set("max-size-buffers", uint(1)); + m_videoQueue.set("max-size-bytes", uint(0)); + m_videoQueue.set("max-size-time", quint64(0)); } void QGstreamerVideoOutput::flushSubtitles() { - if (!subtitleSink.isNull()) { - auto pad = subtitleSink.staticPad("sink"); + if (!m_subtitleSink.isNull()) { + auto pad = m_subtitleSink.staticPad("sink"); auto *event = gst_event_new_flush_start(); pad.sendEvent(event); event = gst_event_new_flush_stop(false); @@ -196,4 +176,28 @@ void QGstreamerVideoOutput::flushSubtitles() } } +void QGstreamerVideoOutput::setNativeSize(QSize sz) +{ + m_nativeSize = sz; + updateNativeSize(); +} + +void QGstreamerVideoOutput::setRotation(QtVideo::Rotation rot) +{ + m_rotation = rot; + updateNativeSize(); +} + +void QGstreamerVideoOutput::updateSubtitle(QString string) +{ + // GStreamer thread + + QMetaObject::invokeMethod(this, [this, string = std::move(string)]() mutable { + m_lastSubtitleString = string; + Q_EMIT subtitleChanged(std::move(string)); + }); +} + QT_END_NAMESPACE + +#include "moc_qgstreamervideooutput_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h index bf6bc5497..a460745f4 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERVIDEOOUTPUT_P_H #define QGSTREAMERVIDEOOUTPUT_P_H @@ -53,53 +17,70 @@ #include <QtCore/qobject.h> #include <private/qtmultimediaglobal_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> +#include <private/qmultimediautils_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgstsubtitlesink_p.h> #include <qwaitcondition.h> #include <qmutex.h> #include <qpointer.h> -#include <qgstreamervideosink_p.h> QT_BEGIN_NAMESPACE class QVideoSink; -class Q_MULTIMEDIA_EXPORT QGstreamerVideoOutput : public QObject +class QGstreamerVideoOutput : public QObject, QAbstractSubtitleObserver { Q_OBJECT public: - QGstreamerVideoOutput(QObject *parent = 0); + static QMaybe<QGstreamerVideoOutput *> create(QObject *parent = nullptr); ~QGstreamerVideoOutput(); void setVideoSink(QVideoSink *sink); - QGstreamerVideoSink *gstreamerVideoSink() const { return m_videoSink; } + QGstreamerVideoSink *gstreamerVideoSink() const { return m_platformVideoSink; } void setPipeline(const QGstPipeline &pipeline); - QGstElement gstElement() const { return gstVideoOutput; } - void linkSubtitleStream(QGstElement subtitleSrc); - void unlinkSubtitleStream(); + QGstElement gstElement() const { return m_outputBin; } + QGstElement gstSubtitleElement() const { return m_subtitleSink; } + + void setActive(bool); void setIsPreview(); void flushSubtitles(); + void setNativeSize(QSize); + void setRotation(QtVideo::Rotation); + + void updateSubtitle(QString) override; + +signals: + void subtitleChanged(QString); + private: - void doLinkSubtitleStream(); + explicit QGstreamerVideoOutput(QObject *parent); + + void updateNativeSize(); - QPointer<QGstreamerVideoSink> m_videoSink; - bool isFakeSink = true; + QPointer<QGstreamerVideoSink> m_platformVideoSink; // Gst elements - QGstPipeline gstPipeline; + QGstPipeline m_pipeline; + + QGstBin m_outputBin; + QGstElement m_videoQueue; + QGstElement m_videoConvertScale; + QGstElement m_videoSink; - QGstBin gstVideoOutput; - QGstElement videoQueue; - QGstElement videoConvert; - QGstElement videoSink; + QGstElement m_subtitleSink; + QMetaObject::Connection m_subtitleConnection; + QString m_lastSubtitleString; - QGstElement subtitleSrc; - QGstElement subtitleSink; + bool m_isActive{ false }; + QSize m_nativeSize; + QtVideo::Rotation m_rotation{}; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp index f83f1d518..6ca23006b 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp @@ -1,70 +1,34 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamervideooverlay_p.h" #include <QtGui/qguiapplication.h> -#include "qgstutils_p.h" -#include "qgst_p.h" -#include "qgstreamermessage_p.h" -#include "qgstreamervideosink_p.h" +#include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include <gst/video/videooverlay.h> +#include <common/qglist_helper_p.h> +#include <common/qgst_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgstutils_p.h> -#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <gst/video/videooverlay.h> QT_BEGIN_NAMESPACE struct ElementMap { - const char *qtPlatform; - const char *gstreamerElement; + QStringView qtPlatform; + const char *gstreamerElement = nullptr; }; // Ordered by descending priority -static constexpr ElementMap elementMap[] = -{ - { "xcb", "xvimagesink" }, - { "xcb", "ximagesink" }, +static constexpr ElementMap elementMap[] = { + { u"xcb", "xvimagesink" }, + { u"xcb", "ximagesink" }, // wayland - { "wayland", "waylandsink" } + { u"wayland", "waylandsink" }, }; static bool qt_gst_element_is_functioning(QGstElement element) @@ -80,13 +44,14 @@ static bool qt_gst_element_is_functioning(QGstElement element) static QGstElement findBestVideoSink() { + using namespace Qt::StringLiterals; QString platform = QGuiApplication::platformName(); // First, try some known video sinks, depending on the Qt platform plugin in use. - for (auto i : elementMap) { - if (platform != QLatin1String(i.qtPlatform)) + for (const auto &i : elementMap) { + if (platform != i.qtPlatform) continue; - QGstElement choice(i.gstreamerElement, i.gstreamerElement); + QGstElement choice = QGstElement::createFromFactory(i.gstreamerElement, i.gstreamerElement); if (choice.isNull()) continue; @@ -96,20 +61,18 @@ static QGstElement findBestVideoSink() // We need a native window ID to use the GstVideoOverlay interface. // Bail out if the Qt platform plugin in use cannot provide a sensible WId. - if (platform != QLatin1String("xcb") && platform != QLatin1String("wayland")) + if (platform != QStringView{ u"xcb" } && platform != QStringView{ u"wayland" }) return {}; QGstElement choice; // If none of the known video sinks are available, try to find one that implements the // GstVideoOverlay interface and has autoplugging rank. GList *list = qt_gst_video_sinks(); - for (GList *item = list; item != nullptr; item = item->next) { - GstElementFactory *f = GST_ELEMENT_FACTORY(item->data); - + for (GstElementFactory *f : QGstUtils::GListRangeAdaptor<GstElementFactory *>(list)) { if (!gst_element_factory_has_interface(f, "GstVideoOverlay")) continue; - choice = QGstElement(gst_element_factory_create(f, nullptr)); + choice = QGstElement::createFromFactory(f, nullptr); if (choice.isNull()) continue; @@ -132,7 +95,7 @@ QGstreamerVideoOverlay::QGstreamerVideoOverlay(QGstreamerVideoSink *parent, cons { QGstElement sink; if (!elementName.isEmpty()) - sink = QGstElement(elementName.constData(), nullptr); + sink = QGstElement::createFromFactory(elementName.constData()); else sink = findBestVideoSink(); @@ -157,7 +120,7 @@ void QGstreamerVideoOverlay::setVideoSink(QGstElement sink) if (sink.isNull()) return; - m_videoSink = sink; + m_videoSink = std::move(sink); QGstPad pad = m_videoSink.staticPad("sink"); addProbeToPad(pad.pad()); @@ -220,7 +183,7 @@ void QGstreamerVideoOverlay::applyRenderRect() void QGstreamerVideoOverlay::probeCaps(GstCaps *caps) { - QSize size = QGstCaps(caps).at(0).resolution(); + QSize size = QGstCaps(caps, QGstCaps::NeedsRef).at(0).resolution(); if (size != m_nativeVideoSize) { m_nativeVideoSize = size; m_gstreamerVideoSink->setNativeSize(m_nativeVideoSize); @@ -244,10 +207,12 @@ void QGstreamerVideoOverlay::setFullScreen(bool fullscreen) bool QGstreamerVideoOverlay::processSyncMessage(const QGstreamerMessage &message) { - if (!gst_is_video_overlay_prepare_window_handle_message(message.rawMessage())) + if (!gst_is_video_overlay_prepare_window_handle_message(message.message())) return false; gst_video_overlay_set_window_handle(GST_VIDEO_OVERLAY(m_videoSink.object()), m_windowId); return true; } QT_END_NAMESPACE + +#include "moc_qgstreamervideooverlay_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h index c58b54d3c..588e8b5e4 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERVIDEOOVERLAY_P_H #define QGSTREAMERVIDEOOVERLAY_P_H @@ -51,22 +15,22 @@ // We mean it. // -#include <qgstpipeline_p.h> -#include <qgstreamerbufferprobe_p.h> -#include <qgst_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamerbufferprobe_p.h> +#include <common/qgst_p.h> #include <QtGui/qwindowdefs.h> QT_BEGIN_NAMESPACE class QGstreamerVideoSink; -class Q_MULTIMEDIA_EXPORT QGstreamerVideoOverlay - : public QObject - , public QGstreamerSyncMessageFilter - , private QGstreamerBufferProbe +class QGstreamerVideoOverlay : public QObject, + public QGstreamerSyncMessageFilter, + private QGstreamerBufferProbe { Q_OBJECT public: - explicit QGstreamerVideoOverlay(QGstreamerVideoSink *parent = 0, const QByteArray &elementName = QByteArray()); + explicit QGstreamerVideoOverlay(QGstreamerVideoSink *parent = nullptr, + const QByteArray &elementName = QByteArray()); virtual ~QGstreamerVideoOverlay(); QGstElement videoSink() const; diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp index dc439f71e..05d1fe0ed 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp @@ -1,124 +1,146 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamervideosink_p.h" -#include "qgstvideorenderersink_p.h" -#include "qgstsubtitlesink_p.h" -#include <qgstutils_p.h> -#include <QtGui/private/qrhi_p.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreamervideosink_p.h> +#include <common/qgstvideorenderersink_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstutils_p.h> +#include <rhi/qrhi.h> + +#include <QtCore/qdebug.h> +#include <QtCore/qloggingcategory.h> #if QT_CONFIG(gstreamer_gl) -#include <QtGui/private/qrhigles2_p.h> -#include <QGuiApplication> -#include <QtGui/qopenglcontext.h> -#include <QWindow> -#include <qpa/qplatformnativeinterface.h> -#include <gst/gl/gstglconfig.h> +# include <QtGui/QGuiApplication> +# include <QtGui/qopenglcontext.h> +# include <QtGui/QWindow> +# include <QtGui/qpa/qplatformnativeinterface.h> +# include <gst/gl/gstglconfig.h> -#if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") +# if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") # include <gst/gl/x11/gstgldisplay_x11.h> -#endif -#if GST_GL_HAVE_PLATFORM_EGL +# endif +# if GST_GL_HAVE_PLATFORM_EGL # include <gst/gl/egl/gstgldisplay_egl.h> # include <EGL/egl.h> # include <EGL/eglext.h> -#endif -#if GST_GL_HAVE_WINDOW_WAYLAND && __has_include("wayland-client.h") +# endif +# if GST_GL_HAVE_WINDOW_WAYLAND && __has_include("wayland-client.h") # include <gst/gl/wayland/gstgldisplay_wayland.h> -#endif +# endif #endif // #if QT_CONFIG(gstreamer_gl) -#include <QtCore/qdebug.h> - -#include <QtCore/qloggingcategory.h> - QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcMediaVideoSink, "qt.multimedia.videosink") +static Q_LOGGING_CATEGORY(qLcGstVideoSink, "qt.multimedia.gstvideosink"); QGstreamerVideoSink::QGstreamerVideoSink(QVideoSink *parent) - : QPlatformVideoSink(parent) + : QPlatformVideoSink{ + parent, + }, + m_sinkBin{ + QGstBin::create("videoSinkBin"), + } { - sinkBin = QGstBin("videoSinkBin"); - // This is a hack for some iMX platforms. Thos require the use of a special video + // This is a hack for some iMX and NVidia platforms. These require the use of a special video // conversion element in the pipeline before the video sink, as they unfortunately - // output some proprietary format from the decoder even though it's marked as + // output some proprietary format from the decoder even though it's sometimes marked as // a regular supported video/x-raw format. // // To fix this, simply insert the element into the pipeline if it's available. Otherwise // we simply use an identity element. - gstQueue = QGstElement("queue"); - auto imxVideoConvert = QGstElement("imxvideoconvert_g2d"); - if (!imxVideoConvert.isNull()) - gstPreprocess = imxVideoConvert; - else - gstPreprocess = QGstElement("identity"); - sinkBin.add(gstQueue, gstPreprocess); - gstQueue.link(gstPreprocess); - sinkBin.addGhostPad(gstQueue, "sink"); - - gstSubtitleSink = GST_ELEMENT(QGstSubtitleSink::createSink(this)); + QGstElementFactoryHandle factory; + + // QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the + // conversion element. Ideally we construct the element programatically, though. + QByteArray preprocessOverride = qgetenv("QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT"); + if (!preprocessOverride.isEmpty()) { + qCDebug(qLcGstVideoSink) << "requesting conversion element from environment:" + << preprocessOverride; + + m_gstPreprocess = QGstBin::createFromPipelineDescription(preprocessOverride, nullptr, + /*ghostUnlinkedPads=*/true); + if (!m_gstPreprocess) + qCWarning(qLcGstVideoSink) << "Cannot create conversion element:" << preprocessOverride; + } + + if (!m_gstPreprocess) { + // This is a hack for some iMX and NVidia platforms. These require the use of a special + // video conversion element in the pipeline before the video sink, as they unfortunately + // output some proprietary format from the decoder even though it's sometimes marked as + // a regular supported video/x-raw format. + static constexpr auto decodersToTest = { + "imxvideoconvert_g2d", + "nvvidconv", + }; + + for (const char *decoder : decodersToTest) { + factory = QGstElement::findFactory(decoder); + if (factory) + break; + } + + if (factory) { + qCDebug(qLcGstVideoSink) + << "instantiating conversion element:" + << g_type_name(gst_element_factory_get_element_type(factory.get())); + + m_gstPreprocess = QGstElement::createFromFactory(factory, "preprocess"); + } + } + + bool disablePixelAspectRatio = + qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO"); + if (disablePixelAspectRatio) { + // Enabling the pixel aspect ratio may expose a gstreamer bug on cameras that don't expose a + // pixel-aspect-ratio via `VIDIOC_CROPCAP`. This can cause the caps negotiation to fail. + // Using the QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO environment variable, one can disable + // pixel-aspect-ratio handling + // + // compare: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6242 + m_gstCapsFilter = + QGstElement::createFromFactory("identity", "nullPixelAspectRatioCapsFilter"); + } else { + m_gstCapsFilter = + QGstElement::createFromFactory("capsfilter", "pixelAspectRatioCapsFilter"); + QGstCaps capsFilterCaps{ + gst_caps_new_simple("video/x-raw", "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL), + QGstCaps::HasRef, + }; + g_object_set(m_gstCapsFilter.element(), "caps", capsFilterCaps.caps(), NULL); + } + + if (m_gstPreprocess) { + m_sinkBin.add(m_gstPreprocess, m_gstCapsFilter); + qLinkGstElements(m_gstPreprocess, m_gstCapsFilter); + m_sinkBin.addGhostPad(m_gstPreprocess, "sink"); + } else { + m_sinkBin.add(m_gstCapsFilter); + m_sinkBin.addGhostPad(m_gstCapsFilter, "sink"); + } } QGstreamerVideoSink::~QGstreamerVideoSink() { - unrefGstContexts(); + emit aboutToBeDestroyed(); - setPipeline(QGstPipeline()); + unrefGstContexts(); } QGstElement QGstreamerVideoSink::gstSink() { updateSinkElement(); - return sinkBin; + return m_sinkBin; } -void QGstreamerVideoSink::setPipeline(QGstPipeline pipeline) +void QGstreamerVideoSink::setActive(bool isActive) { - gstPipeline = pipeline; -} + if (m_isActive == isActive) + return; + m_isActive = isActive; -bool QGstreamerVideoSink::inStoppedState() const -{ - if (gstPipeline.isNull()) - return true; - return gstPipeline.inStoppedState(); + if (m_gstQtSink) + m_gstQtSink.setActive(isActive); } void QGstreamerVideoSink::setRhi(QRhi *rhi) @@ -130,59 +152,65 @@ void QGstreamerVideoSink::setRhi(QRhi *rhi) m_rhi = rhi; updateGstContexts(); - if (!gstQtSink.isNull()) { - // force creation of a new sink with proper caps + if (!m_gstQtSink.isNull()) { + // force creation of a new sink with proper caps. createQtSink(); updateSinkElement(); + + QGstPipeline pipeline = m_sinkBin.getPipeline(); + if (pipeline) + pipeline.flush(); // the caps may change, so we need to flush the pipeline. } } void QGstreamerVideoSink::createQtSink() { - gstQtSink = QGstElement(reinterpret_cast<GstElement *>(QGstVideoRendererSink::createSink(this))); + if (m_gstQtSink) + m_gstQtSink.setStateSync(GST_STATE_NULL); + + m_gstQtSink = QGstVideoRendererSink::createSink(this); + if (m_gstQtSink) + m_gstQtSink.setActive(m_isActive); } void QGstreamerVideoSink::updateSinkElement() { QGstElement newSink; - if (gstQtSink.isNull()) + if (m_gstQtSink.isNull()) createQtSink(); - newSink = gstQtSink; - if (newSink == gstVideoSink) + newSink = m_gstQtSink; + + if (newSink == m_gstVideoSink) return; - gstPipeline.beginConfig(); + QGstPipeline::modifyPipelineWhileNotRunning(m_sinkBin.getPipeline(), [&] { + if (!m_gstVideoSink.isNull()) + m_sinkBin.stopAndRemoveElements(m_gstVideoSink); - if (!gstVideoSink.isNull()) { - gstVideoSink.setStateSync(GST_STATE_NULL); - sinkBin.remove(gstVideoSink); - } + newSink.set("async", false); // no asynchronous state changes - gstVideoSink = newSink; - sinkBin.add(gstVideoSink); - if (!gstPreprocess.link(gstVideoSink)) - qCDebug(qLcMediaVideoSink) << "couldn't link preprocess and sink"; - gstVideoSink.setState(GST_STATE_PAUSED); + m_gstVideoSink = newSink; + m_sinkBin.add(m_gstVideoSink); + qLinkGstElements(m_gstCapsFilter, m_gstVideoSink); + m_gstVideoSink.syncStateWithParent(); + }); - gstPipeline.endConfig(); - gstPipeline.dumpGraph("updateVideoSink"); + m_sinkBin.dumpPipelineGraph("updateVideoSink"); } void QGstreamerVideoSink::unrefGstContexts() { - if (m_gstGlDisplayContext) - gst_context_unref(m_gstGlDisplayContext); - m_gstGlDisplayContext = nullptr; - if (m_gstGlLocalContext) - gst_context_unref(m_gstGlLocalContext); - m_gstGlLocalContext = nullptr; + m_gstGlDisplayContext.close(); + m_gstGlLocalContext.close(); m_eglDisplay = nullptr; m_eglImageTargetTexture2D = nullptr; } void QGstreamerVideoSink::updateGstContexts() { + using namespace Qt::Literals; + unrefGstContexts(); #if QT_CONFIG(gstreamer_gl) @@ -195,34 +223,38 @@ void QGstreamerVideoSink::updateGstContexts() const QString platform = QGuiApplication::platformName(); QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface(); - m_eglDisplay = pni->nativeResourceForIntegration("egldisplay"); + m_eglDisplay = pni->nativeResourceForIntegration("egldisplay"_ba); // qDebug() << "platform is" << platform << m_eglDisplay; - GstGLDisplay *gstGlDisplay = nullptr; - const char *contextName = "eglcontext"; + QGstGLDisplayHandle gstGlDisplay; + + QByteArray contextName = "eglcontext"_ba; GstGLPlatform glPlatform = GST_GL_PLATFORM_EGL; // use the egl display if we have one if (m_eglDisplay) { #if GST_GL_HAVE_PLATFORM_EGL - gstGlDisplay = (GstGLDisplay *)gst_gl_display_egl_new_with_egl_display(m_eglDisplay); + gstGlDisplay.reset( + GST_GL_DISPLAY_CAST(gst_gl_display_egl_new_with_egl_display(m_eglDisplay))); m_eglImageTargetTexture2D = eglGetProcAddress("glEGLImageTargetTexture2DOES"); #endif } else { - auto display = pni->nativeResourceForIntegration("display"); + auto display = pni->nativeResourceForIntegration("display"_ba); if (display) { #if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") if (platform == QLatin1String("xcb")) { - contextName = "glxcontext"; + contextName = "glxcontext"_ba; glPlatform = GST_GL_PLATFORM_GLX; - gstGlDisplay = (GstGLDisplay *)gst_gl_display_x11_new_with_display((Display *)display); + gstGlDisplay.reset(GST_GL_DISPLAY_CAST( + gst_gl_display_x11_new_with_display(reinterpret_cast<Display *>(display)))); } #endif #if GST_GL_HAVE_WINDOW_WAYLAND && __has_include("wayland-client.h") if (platform.startsWith(QLatin1String("wayland"))) { Q_ASSERT(!gstGlDisplay); - gstGlDisplay = (GstGLDisplay *)gst_gl_display_wayland_new_with_display((struct wl_display *)display); + gstGlDisplay.reset(GST_GL_DISPLAY_CAST(gst_gl_display_wayland_new_with_display( + reinterpret_cast<struct wl_display *>(display)))); } #endif } @@ -238,33 +270,43 @@ void QGstreamerVideoSink::updateGstContexts() qWarning() << "Could not find resource for" << contextName; GstGLAPI glApi = QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGL ? GST_GL_API_OPENGL : GST_GL_API_GLES2; - GstGLContext *appContext = gst_gl_context_new_wrapped(gstGlDisplay, (guintptr)nativeContext, glPlatform, glApi); + QGstGLContextHandle appContext{ + gst_gl_context_new_wrapped(gstGlDisplay.get(), guintptr(nativeContext), glPlatform, glApi), + }; if (!appContext) qWarning() << "Could not create wrappped context for platform:" << glPlatform; - GstGLContext *displayContext = nullptr; - GError *error = nullptr; - gst_gl_display_create_context(gstGlDisplay, appContext, &displayContext, &error); + gst_gl_context_activate(appContext.get(), true); + + QUniqueGErrorHandle error; + gst_gl_context_fill_info(appContext.get(), &error); if (error) { - qWarning() << "Could not create display context:" << error->message; - g_clear_error(&error); + qWarning() << "Could not fill context info:" << error; + error = {}; } - if (appContext) - gst_object_unref(appContext); + QGstGLContextHandle displayContext; + gst_gl_display_create_context(gstGlDisplay.get(), appContext.get(), &displayContext, &error); + if (error) + qWarning() << "Could not create display context:" << error; + + appContext.close(); - m_gstGlDisplayContext = gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, false); - gst_context_set_gl_display(m_gstGlDisplayContext, gstGlDisplay); - gst_object_unref(gstGlDisplay); + m_gstGlDisplayContext.reset(gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, false)); + gst_context_set_gl_display(m_gstGlDisplayContext.get(), gstGlDisplay.get()); - m_gstGlLocalContext = gst_context_new("gst.gl.local_context", false); - GstStructure *structure = gst_context_writable_structure(m_gstGlLocalContext); - gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext, nullptr); - gst_object_unref(displayContext); + m_gstGlLocalContext.reset(gst_context_new("gst.gl.local_context", false)); + GstStructure *structure = gst_context_writable_structure(m_gstGlLocalContext.get()); + gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext.get(), nullptr); + displayContext.close(); - if (!gstPipeline.isNull()) - gst_element_set_context(gstPipeline.element(), m_gstGlLocalContext); + QGstPipeline pipeline = m_sinkBin.getPipeline(); + + if (pipeline) + gst_element_set_context(pipeline.element(), m_gstGlLocalContext.get()); #endif // #if QT_CONFIG(gstreamer_gl) } QT_END_NAMESPACE + +#include "moc_qgstreamervideosink_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h index 216dd61ee..b892ac752 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h @@ -1,44 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QGSTREAMERVIDEOWINDOW_H -#define QGSTREAMERVIDEOWINDOW_H +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGSTREAMERVIDEOSINK_H +#define QGSTREAMERVIDEOSINK_H // // W A R N I N G @@ -51,44 +15,37 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qplatformvideosink_p.h> - -#include <qgstpipeline_p.h> -#include <qgstreamervideooverlay_p.h> -#include <QtGui/qcolor.h> -#include <qvideosink.h> +#include <QtMultimedia/qvideosink.h> +#include <QtMultimedia/private/qplatformvideosink_p.h> -#if QT_CONFIG(gstreamer_gl) -#include <gst/gl/gl.h> -#endif +#include <common/qgstvideorenderersink_p.h> +#include <common/qgstpipeline_p.h> QT_BEGIN_NAMESPACE -class QGstreamerVideoRenderer; -class QVideoWindow; -class Q_MULTIMEDIA_EXPORT QGstreamerVideoSink - : public QPlatformVideoSink +class QGstreamerVideoSink : public QPlatformVideoSink { Q_OBJECT + public: - explicit QGstreamerVideoSink(QVideoSink *parent = 0); + explicit QGstreamerVideoSink(QVideoSink *parent = nullptr); ~QGstreamerVideoSink(); void setRhi(QRhi *rhi) override; QRhi *rhi() const { return m_rhi; } QGstElement gstSink(); - QGstElement subtitleSink() const { return gstSubtitleSink; } - void setPipeline(QGstPipeline pipeline); - bool inStoppedState() const; - - GstContext *gstGlDisplayContext() const { return m_gstGlDisplayContext; } - GstContext *gstGlLocalContext() const { return m_gstGlLocalContext; } + GstContext *gstGlDisplayContext() const { return m_gstGlDisplayContext.get(); } + GstContext *gstGlLocalContext() const { return m_gstGlLocalContext.get(); } Qt::HANDLE eglDisplay() const { return m_eglDisplay; } QFunctionPointer eglImageTargetTexture2D() const { return m_eglImageTargetTexture2D; } + void setActive(bool); + +Q_SIGNALS: + void aboutToBeDestroyed(); + private: void createQtSink(); void updateSinkElement(); @@ -96,20 +53,20 @@ private: void unrefGstContexts(); void updateGstContexts(); - QGstPipeline gstPipeline; - QGstBin sinkBin; - QGstElement gstQueue; - QGstElement gstPreprocess; - QGstElement gstVideoSink; - QGstElement gstQtSink; - QGstElement gstSubtitleSink; + QGstBin m_sinkBin; + QGstElement m_gstPreprocess; + QGstElement m_gstCapsFilter; + QGstElement m_gstVideoSink; + QGstVideoRendererSinkElement m_gstQtSink; QRhi *m_rhi = nullptr; + bool m_isActive = true; Qt::HANDLE m_eglDisplay = nullptr; QFunctionPointer m_eglImageTargetTexture2D = nullptr; - GstContext *m_gstGlLocalContext = nullptr; - GstContext *m_gstGlDisplayContext = nullptr; + + QGstContextHandle m_gstGlLocalContext; + QGstContextHandle m_gstGlDisplayContext; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp index d9b76d57f..58b5c3f53 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp @@ -1,94 +1,64 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QDebug> -#include <QThread> -#include <QEvent> - -#include "qgstreamervideosink_p.h" +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + #include "qgstsubtitlesink_p.h" -#include "qgstutils_p.h" +#include "qgst_debug_p.h" + +#include <QtCore/qdebug.h> QT_BEGIN_NAMESPACE -static GstBaseSinkClass *sink_parent_class; -static thread_local QGstreamerVideoSink *current_sink; +namespace { +GstBaseSinkClass *gst_sink_parent_class; +thread_local QAbstractSubtitleObserver *gst_current_observer; + +class QGstSubtitleSinkClass +{ +public: + GstBaseSinkClass parent_class; +}; + +} // namespace #define ST_SINK(s) QGstSubtitleSink *sink(reinterpret_cast<QGstSubtitleSink *>(s)) -QGstSubtitleSink *QGstSubtitleSink::createSink(QGstreamerVideoSink *sink) +QGstElement QGstSubtitleSink::createSink(QAbstractSubtitleObserver *observer) { - current_sink = sink; + gst_current_observer = observer; QGstSubtitleSink *gstSink = reinterpret_cast<QGstSubtitleSink *>( g_object_new(QGstSubtitleSink::get_type(), nullptr)); g_object_set(gstSink, "async", false, nullptr); - return gstSink; + return QGstElement{ + qGstCheckedCast<GstElement>(gstSink), + QGstElement::NeedsRef, + }; } GType QGstSubtitleSink::get_type() { - static GType type = 0; - - if (type == 0) { - static const GTypeInfo info = - { - sizeof(QGstSubtitleSinkClass), // class_size - base_init, // base_init - nullptr, // base_finalize - class_init, // class_init - nullptr, // class_finalize - nullptr, // class_data - sizeof(QGstSubtitleSink), // instance_size - 0, // n_preallocs - instance_init, // instance_init - nullptr // value_table - }; - - type = g_type_register_static( + // clang-format off + static constexpr GTypeInfo info = + { + sizeof(QGstSubtitleSinkClass), // class_size + base_init, // base_init + nullptr, // base_finalize + class_init, // class_init + nullptr, // class_finalize + nullptr, // class_data + sizeof(QGstSubtitleSink), // instance_size + 0, // n_preallocs + instance_init, // instance_init + nullptr // value_table + }; + // clang-format on + + static const GType type = []() { + const auto result = g_type_register_static( GST_TYPE_BASE_SINK, "QGstSubtitleSink", &info, GTypeFlags(0)); - - // Register the sink type to be used in custom piplines. - // When surface is ready the sink can be used. - gst_element_register(nullptr, "qtsubtitlesink", GST_RANK_PRIMARY, type); - } + return result; + }(); return type; } @@ -97,7 +67,7 @@ void QGstSubtitleSink::class_init(gpointer g_class, gpointer class_data) { Q_UNUSED(class_data); - sink_parent_class = reinterpret_cast<GstBaseSinkClass *>(g_type_class_peek_parent(g_class)); + gst_sink_parent_class = reinterpret_cast<GstBaseSinkClass *>(g_type_class_peek_parent(g_class)); GstBaseSinkClass *base_sink_class = reinterpret_cast<GstBaseSinkClass *>(g_class); base_sink_class->render = QGstSubtitleSink::render; @@ -120,57 +90,56 @@ void QGstSubtitleSink::class_init(gpointer g_class, gpointer class_data) void QGstSubtitleSink::base_init(gpointer g_class) { - static GstStaticPadTemplate sink_pad_template = GST_STATIC_PAD_TEMPLATE( - "sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS("ANY")); + static GstStaticPadTemplate sink_pad_template = + GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS("ANY")); gst_element_class_add_pad_template( GST_ELEMENT_CLASS(g_class), gst_static_pad_template_get(&sink_pad_template)); } -void QGstSubtitleSink::instance_init(GTypeInstance *instance, gpointer g_class) +void QGstSubtitleSink::instance_init(GTypeInstance *instance, gpointer /*g_class*/) { - Q_UNUSED(g_class); ST_SINK(instance); - Q_ASSERT(current_sink); - sink->sink = current_sink; - current_sink = nullptr; + Q_ASSERT(gst_current_observer); + sink->observer = gst_current_observer; + gst_current_observer = nullptr; } void QGstSubtitleSink::finalize(GObject *object) { // Chain up - G_OBJECT_CLASS(sink_parent_class)->finalize(object); + G_OBJECT_CLASS(gst_sink_parent_class)->finalize(object); } GstStateChangeReturn QGstSubtitleSink::change_state(GstElement *element, GstStateChange transition) { - return GST_ELEMENT_CLASS(sink_parent_class)->change_state(element, transition); + return GST_ELEMENT_CLASS(gst_sink_parent_class)->change_state(element, transition); } GstCaps *QGstSubtitleSink::get_caps(GstBaseSink *base, GstCaps *filter) { - return sink_parent_class->get_caps(base, filter); + return gst_sink_parent_class->get_caps(base, filter); } gboolean QGstSubtitleSink::set_caps(GstBaseSink *base, GstCaps *caps) { - qDebug() << "set_caps:" << QGstCaps(caps).toString(); - return sink_parent_class->set_caps(base, caps); + qDebug() << "set_caps:" << caps; + return gst_sink_parent_class->set_caps(base, caps); } gboolean QGstSubtitleSink::propose_allocation(GstBaseSink *base, GstQuery *query) { - return sink_parent_class->propose_allocation(base, query); + return gst_sink_parent_class->propose_allocation(base, query); } GstFlowReturn QGstSubtitleSink::wait_event(GstBaseSink *base, GstEvent *event) { - GstFlowReturn retval = sink_parent_class->wait_event(base, event); + GstFlowReturn retval = gst_sink_parent_class->wait_event(base, event); ST_SINK(base); if (event->type == GST_EVENT_GAP) { -// qDebug() << "gap, clearing subtitle"; - sink->sink->setSubtitleText(QString()); + // qDebug() << "gap, clearing subtitle"; + sink->observer->updateSubtitle(QString()); } return retval; } @@ -182,10 +151,10 @@ GstFlowReturn QGstSubtitleSink::render(GstBaseSink *base, GstBuffer *buffer) GstMapInfo info; QString subtitle; if (gst_memory_map(mem, &info, GST_MAP_READ)) - subtitle = QString::fromUtf8(info.data); + subtitle = QString::fromUtf8(reinterpret_cast<const char *>(info.data)); gst_memory_unmap(mem, &info); // qDebug() << "render" << buffer << subtitle; - sink->sink->setSubtitleText(subtitle); + sink->observer->updateSubtitle(subtitle); return GST_FLOW_OK; } diff --git a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h index 179b02a50..1970ac48b 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTSUBTITLESINK_P_H #define QGSTSUBTITLESINK_P_H @@ -53,24 +17,25 @@ #include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include <QtCore/qlist.h> -#include <QtCore/qmutex.h> -#include <QtCore/qqueue.h> -#include <QtCore/qpointer.h> -#include <QtCore/qwaitcondition.h> -#include <qgst_p.h> +#include <QtCore/qstring.h> +#include <common/qgst_p.h> #include <gst/base/gstbasesink.h> QT_BEGIN_NAMESPACE -class QGstreamerVideoSink; +class QAbstractSubtitleObserver +{ +public: + virtual ~QAbstractSubtitleObserver() = default; + virtual void updateSubtitle(QString) = 0; +}; -class Q_MULTIMEDIA_EXPORT QGstSubtitleSink +class QGstSubtitleSink { public: - GstBaseSink parent; + GstBaseSink parent{}; - static QGstSubtitleSink *createSink(QGstreamerVideoSink *sink); + static QGstElement createSink(QAbstractSubtitleObserver *observer); private: static GType get_type(); @@ -91,14 +56,7 @@ private: static GstFlowReturn render(GstBaseSink *sink, GstBuffer *buffer); private: - QGstreamerVideoSink *sink = nullptr; -}; - - -class QGstSubtitleSinkClass -{ -public: - GstBaseSinkClass parent_class; + QAbstractSubtitleObserver *observer = nullptr; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstutils.cpp b/src/plugins/multimedia/gstreamer/common/qgstutils.cpp index 21173a5cd..8ec2bde3c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstutils.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstutils.cpp @@ -1,70 +1,18 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -#include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include "qgstutils_p.h" +#include <common/qgstutils_p.h> +#include <common/qgst_p.h> -#include <QtCore/qdatetime.h> -#include <QtCore/qdir.h> -#include <QtCore/qbytearray.h> -#include <QtCore/qvariant.h> -#include <QtCore/qregularexpression.h> -#include <QtCore/qsize.h> -#include <QtCore/qset.h> -#include <QtCore/qstringlist.h> -#include <QtGui/qimage.h> -#include <qaudioformat.h> -#include <QtCore/qelapsedtimer.h> -#include <QtMultimedia/qvideoframeformat.h> -#include <private/qmultimediautils_p.h> +#include <QtMultimedia/qaudioformat.h> -#include <gst/audio/audio.h> -#include <gst/video/video.h> - -template<typename T, int N> constexpr int lengthOf(const T (&)[N]) { return N; } +#include <chrono> QT_BEGIN_NAMESPACE - namespace { -static const char *audioSampleFormatNames[QAudioFormat::NSampleFormats] = { +const char *audioSampleFormatNames[QAudioFormat::NSampleFormats] = { nullptr, #if Q_BYTE_ORDER == Q_LITTLE_ENDIAN "U8", @@ -79,7 +27,7 @@ static const char *audioSampleFormatNames[QAudioFormat::NSampleFormats] = { #endif }; -static QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) +QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) { if (fmt) { for (int i = 1; i < QAudioFormat::NSampleFormats; ++i) { @@ -91,7 +39,7 @@ static QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) return QAudioFormat::Unknown; } -} +} // namespace /* Returns audio format for a sample \a sample. @@ -99,16 +47,16 @@ static QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) */ QAudioFormat QGstUtils::audioFormatForSample(GstSample *sample) { - QGstCaps caps = gst_sample_get_caps(sample); + auto caps = QGstCaps(gst_sample_get_caps(sample), QGstCaps::NeedsRef); if (caps.isNull()) - return QAudioFormat(); + return {}; return audioFormatForCaps(caps); } -QAudioFormat QGstUtils::audioFormatForCaps(QGstCaps caps) +QAudioFormat QGstUtils::audioFormatForCaps(const QGstCaps &caps) { QAudioFormat format; - QGstStructure s = caps.at(0); + QGstStructureView s = caps.at(0); if (s.name() != "audio/x-raw") return format; @@ -132,19 +80,21 @@ QAudioFormat QGstUtils::audioFormatForCaps(QGstCaps caps) \note Caller must unreference GstCaps. */ -QGstMutableCaps QGstUtils::capsForAudioFormat(const QAudioFormat &format) +QGstCaps QGstUtils::capsForAudioFormat(const QAudioFormat &format) { if (!format.isValid()) return {}; auto sampleFormat = format.sampleFormat(); - return gst_caps_new_simple( + auto caps = gst_caps_new_simple( "audio/x-raw", "format" , G_TYPE_STRING, audioSampleFormatNames[sampleFormat], "rate" , G_TYPE_INT , format.sampleRate(), "channels", G_TYPE_INT , format.channelCount(), "layout" , G_TYPE_STRING, "interleaved", nullptr); + + return QGstCaps(caps, QGstCaps::HasRef); } QList<QAudioFormat::SampleFormat> QGValue::getSampleFormats() const @@ -155,346 +105,37 @@ QList<QAudioFormat::SampleFormat> QGValue::getSampleFormats() const QList<QAudioFormat::SampleFormat> formats; guint nFormats = gst_value_list_get_size(value); for (guint f = 0; f < nFormats; ++f) { - QGValue v = gst_value_list_get_value(value, f); + QGValue v = QGValue{ gst_value_list_get_value(value, f) }; auto *name = v.toString(); QAudioFormat::SampleFormat fmt = gstSampleFormatToSampleFormat(name); if (fmt == QAudioFormat::Unknown) - continue;; + continue; formats.append(fmt); } return formats; } -namespace { - -struct VideoFormat -{ - QVideoFrameFormat::PixelFormat pixelFormat; - GstVideoFormat gstFormat; -}; - -static const VideoFormat qt_videoFormatLookup[] = -{ - { QVideoFrameFormat::Format_YUV420P, GST_VIDEO_FORMAT_I420 }, - { QVideoFrameFormat::Format_YUV422P, GST_VIDEO_FORMAT_Y42B }, - { QVideoFrameFormat::Format_YV12 , GST_VIDEO_FORMAT_YV12 }, - { QVideoFrameFormat::Format_UYVY , GST_VIDEO_FORMAT_UYVY }, - { QVideoFrameFormat::Format_YUYV , GST_VIDEO_FORMAT_YUY2 }, - { QVideoFrameFormat::Format_NV12 , GST_VIDEO_FORMAT_NV12 }, - { QVideoFrameFormat::Format_NV21 , GST_VIDEO_FORMAT_NV21 }, - { QVideoFrameFormat::Format_AYUV , GST_VIDEO_FORMAT_AYUV }, - { QVideoFrameFormat::Format_Y8 , GST_VIDEO_FORMAT_GRAY8 }, - { QVideoFrameFormat::Format_XRGB8888 , GST_VIDEO_FORMAT_xRGB }, - { QVideoFrameFormat::Format_XBGR8888 , GST_VIDEO_FORMAT_xBGR }, - { QVideoFrameFormat::Format_RGBX8888 , GST_VIDEO_FORMAT_RGBx }, - { QVideoFrameFormat::Format_BGRX8888 , GST_VIDEO_FORMAT_BGRx }, - { QVideoFrameFormat::Format_ARGB8888, GST_VIDEO_FORMAT_ARGB }, - { QVideoFrameFormat::Format_ABGR8888, GST_VIDEO_FORMAT_ABGR }, - { QVideoFrameFormat::Format_RGBA8888, GST_VIDEO_FORMAT_RGBA }, - { QVideoFrameFormat::Format_BGRA8888, GST_VIDEO_FORMAT_BGRA }, -#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN - { QVideoFrameFormat::Format_Y16 , GST_VIDEO_FORMAT_GRAY16_LE }, - { QVideoFrameFormat::Format_P010 , GST_VIDEO_FORMAT_P010_10LE }, -#else - { QVideoFrameFormat::Format_Y16 , GST_VIDEO_FORMAT_GRAY16_BE }, - { QVideoFrameFormat::Format_P010 , GST_VIDEO_FORMAT_P010_10BE }, -#endif -}; - -static int indexOfVideoFormat(QVideoFrameFormat::PixelFormat format) -{ - for (int i = 0; i < lengthOf(qt_videoFormatLookup); ++i) - if (qt_videoFormatLookup[i].pixelFormat == format) - return i; - - return -1; -} - -static int indexOfVideoFormat(GstVideoFormat format) -{ - for (int i = 0; i < lengthOf(qt_videoFormatLookup); ++i) - if (qt_videoFormatLookup[i].gstFormat == format) - return i; - - return -1; -} - -} - -QVideoFrameFormat QGstCaps::formatForCaps(GstVideoInfo *info) const -{ - GstVideoInfo vidInfo; - GstVideoInfo *infoPtr = info ? info : &vidInfo; - - if (gst_video_info_from_caps(infoPtr, caps)) { - int index = indexOfVideoFormat(infoPtr->finfo->format); - - if (index != -1) { - QVideoFrameFormat format( - QSize(infoPtr->width, infoPtr->height), - qt_videoFormatLookup[index].pixelFormat); - - if (infoPtr->fps_d > 0) - format.setFrameRate(qreal(infoPtr->fps_n) / infoPtr->fps_d); - - QVideoFrameFormat::ColorRange range = QVideoFrameFormat::ColorRange_Unknown; - switch (infoPtr->colorimetry.range) { - case GST_VIDEO_COLOR_RANGE_UNKNOWN: - break; - case GST_VIDEO_COLOR_RANGE_0_255: - range = QVideoFrameFormat::ColorRange_Full; - break; - case GST_VIDEO_COLOR_RANGE_16_235: - range = QVideoFrameFormat::ColorRange_Video; - break; - } - format.setColorRange(range); - - QVideoFrameFormat::ColorSpace colorSpace = QVideoFrameFormat::ColorSpace_Undefined; - switch (infoPtr->colorimetry.matrix) { - case GST_VIDEO_COLOR_MATRIX_UNKNOWN: - case GST_VIDEO_COLOR_MATRIX_RGB: - case GST_VIDEO_COLOR_MATRIX_FCC: - break; - case GST_VIDEO_COLOR_MATRIX_BT709: - colorSpace = QVideoFrameFormat::ColorSpace_BT709; - break; - case GST_VIDEO_COLOR_MATRIX_BT601: - colorSpace = QVideoFrameFormat::ColorSpace_BT601; - break; - case GST_VIDEO_COLOR_MATRIX_SMPTE240M: - colorSpace = QVideoFrameFormat::ColorSpace_AdobeRgb; - break; - case GST_VIDEO_COLOR_MATRIX_BT2020: - colorSpace = QVideoFrameFormat::ColorSpace_BT2020; - break; - } - format.setColorSpace(colorSpace); - - QVideoFrameFormat::ColorTransfer transfer = QVideoFrameFormat::ColorTransfer_Unknown; - switch (infoPtr->colorimetry.transfer) { - case GST_VIDEO_TRANSFER_UNKNOWN: - break; - case GST_VIDEO_TRANSFER_GAMMA10: - transfer = QVideoFrameFormat::ColorTransfer_Linear; - break; - case GST_VIDEO_TRANSFER_GAMMA22: - case GST_VIDEO_TRANSFER_SMPTE240M: - case GST_VIDEO_TRANSFER_SRGB: - case GST_VIDEO_TRANSFER_ADOBERGB: - transfer = QVideoFrameFormat::ColorTransfer_Gamma22; - break; - case GST_VIDEO_TRANSFER_GAMMA18: - case GST_VIDEO_TRANSFER_GAMMA20: - // not quite, but best fit - case GST_VIDEO_TRANSFER_BT709: - case GST_VIDEO_TRANSFER_BT2020_12: - transfer = QVideoFrameFormat::ColorTransfer_BT709; - break; - case GST_VIDEO_TRANSFER_GAMMA28: - transfer = QVideoFrameFormat::ColorTransfer_Gamma28; - break; - case GST_VIDEO_TRANSFER_LOG100: - case GST_VIDEO_TRANSFER_LOG316: - break; -#if GST_CHECK_VERSION(1, 18, 0) - case GST_VIDEO_TRANSFER_SMPTE2084: - transfer = QVideoFrameFormat::ColorTransfer_ST2084; - break; - case GST_VIDEO_TRANSFER_ARIB_STD_B67: - transfer = QVideoFrameFormat::ColorTransfer_STD_B67; - break; - case GST_VIDEO_TRANSFER_BT2020_10: - transfer = QVideoFrameFormat::ColorTransfer_BT709; - break; - case GST_VIDEO_TRANSFER_BT601: - transfer = QVideoFrameFormat::ColorTransfer_BT601; - break; -#endif - } - format.setColorTransfer(transfer); - - return format; - } - } - return QVideoFrameFormat(); -} - -void QGstMutableCaps::addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, const char *modifier) -{ - GValue list = {}; - g_value_init(&list, GST_TYPE_LIST); - - for (QVideoFrameFormat::PixelFormat format : formats) { - int index = indexOfVideoFormat(format); - if (index == -1) - continue; - GValue item = {}; - - g_value_init(&item, G_TYPE_STRING); - g_value_set_string(&item, gst_video_format_to_string(qt_videoFormatLookup[index].gstFormat)); - gst_value_list_append_value(&list, &item); - g_value_unset(&item); - } - QGValue v(&list); - auto *structure = gst_structure_new("video/x-raw", - "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, INT_MAX, 1, - "width" , GST_TYPE_INT_RANGE, 1, INT_MAX, - "height" , GST_TYPE_INT_RANGE, 1, INT_MAX, - nullptr); - gst_structure_set_value(structure, "format", &list); - gst_caps_append_structure(caps, structure); - g_value_unset(&list); - - if (modifier) - gst_caps_set_features(caps, size() - 1, gst_caps_features_from_string(modifier)); -} - -QGstMutableCaps QGstMutableCaps::fromCameraFormat(const QCameraFormat &format) +void QGstUtils::setFrameTimeStampsFromBuffer(QVideoFrame *frame, GstBuffer *buffer) { - QGstMutableCaps caps; - caps.create(); - - QSize size = format.resolution(); - GstStructure *structure = nullptr; -// int num = 0; -// int den = 1; -// if (format.maxFrameRate() > 0) -// qt_real_to_fraction(1. / format.maxFrameRate(), &num, &den); -// qDebug() << "fromCameraFormat" << format.maxFrameRate() << num << den; - - if (format.pixelFormat() == QVideoFrameFormat::Format_Jpeg) { - structure = gst_structure_new("image/jpeg", - "width" , G_TYPE_INT, size.width(), - "height" , G_TYPE_INT, size.height(), -// "framerate", GST_TYPE_FRACTION, den, num, - nullptr); - } else { - int index = indexOfVideoFormat(format.pixelFormat()); - if (index < 0) - return QGstMutableCaps(); - auto gstFormat = qt_videoFormatLookup[index].gstFormat; - structure = gst_structure_new("video/x-raw", - "format" , G_TYPE_STRING, gst_video_format_to_string(gstFormat), - "width" , G_TYPE_INT, size.width(), - "height" , G_TYPE_INT, size.height(), -// "framerate", GST_TYPE_FRACTION, den, num, - nullptr); - } - gst_caps_append_structure(caps.caps, structure); - return caps; -} + using namespace std::chrono; + using namespace std::chrono_literals; -void QGstUtils::setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer) -{ // GStreamer uses nanoseconds, Qt uses microseconds - qint64 startTime = GST_BUFFER_TIMESTAMP(buffer); - if (startTime >= 0) { - frame->setStartTime(startTime/G_GINT64_CONSTANT (1000)); - - qint64 duration = GST_BUFFER_DURATION(buffer); - if (duration >= 0) - frame->setEndTime((startTime + duration)/G_GINT64_CONSTANT (1000)); - } -} - -QSize QGstStructure::resolution() const -{ - QSize size; + nanoseconds startTime{ GST_BUFFER_TIMESTAMP(buffer) }; + if (startTime >= 0ns) { + frame->setStartTime(floor<microseconds>(startTime).count()); - int w, h; - if (structure && - gst_structure_get_int(structure, "width", &w) && - gst_structure_get_int(structure, "height", &h)) { - size.rwidth() = w; - size.rheight() = h; + nanoseconds duration{ GST_BUFFER_DURATION(buffer) }; + if (duration >= 0ns) + frame->setEndTime(floor<microseconds>(startTime + duration).count()); } - - return size; -} - -QVideoFrameFormat::PixelFormat QGstStructure::pixelFormat() const -{ - QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid; - - if (!structure) - return pixelFormat; - - if (gst_structure_has_name(structure, "video/x-raw")) { - const gchar *s = gst_structure_get_string(structure, "format"); - if (s) { - GstVideoFormat format = gst_video_format_from_string(s); - int index = indexOfVideoFormat(format); - - if (index != -1) - pixelFormat = qt_videoFormatLookup[index].pixelFormat; - } - } else if (gst_structure_has_name(structure, "image/jpeg")) { - pixelFormat = QVideoFrameFormat::Format_Jpeg; - } - - return pixelFormat; -} - -QGRange<float> QGstStructure::frameRateRange() const -{ - float minRate = 0.; - float maxRate = 0.; - - if (!structure) - return {0.f, 0.f}; - - auto extractFraction = [] (const GValue *v) -> float { - return (float)gst_value_get_fraction_numerator(v)/(float)gst_value_get_fraction_denominator(v); - }; - auto extractFrameRate = [&] (const GValue *v) { - auto insert = [&] (float min, float max) { - if (max > maxRate) - maxRate = max; - if (min < minRate) - minRate = min; - }; - - if (GST_VALUE_HOLDS_FRACTION(v)) { - float rate = extractFraction(v); - insert(rate, rate); - } else if (GST_VALUE_HOLDS_FRACTION_RANGE(v)) { - auto *min = gst_value_get_fraction_range_max(v); - auto *max = gst_value_get_fraction_range_max(v); - insert(extractFraction(min), extractFraction(max)); - } - }; - - const GValue *gstFrameRates = gst_structure_get_value(structure, "framerate"); - if (gstFrameRates) { - if (GST_VALUE_HOLDS_LIST(gstFrameRates)) { - guint nFrameRates = gst_value_list_get_size(gstFrameRates); - for (guint f = 0; f < nFrameRates; ++f) { - extractFrameRate(gst_value_list_get_value(gstFrameRates, f)); - } - } else { - extractFrameRate(gstFrameRates); - } - } else { - const GValue *min = gst_structure_get_value(structure, "min-framerate"); - const GValue *max = gst_structure_get_value(structure, "max-framerate"); - if (min && max) { - minRate = extractFraction(min); - maxRate = extractFraction(max); - } - } - - return {minRate, maxRate}; } GList *qt_gst_video_sinks() { - GList *list = nullptr; - - list = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, + return gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_SINK + | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL); - - return list; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstutils_p.h b/src/plugins/multimedia/gstreamer/common/qgstutils_p.h index 9782486a7..c65fcf090 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstutils_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstutils_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTUTILS_P_H #define QGSTUTILS_P_H @@ -51,34 +15,26 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <QtCore/qlist.h> -#include <QtCore/qmap.h> -#include <QtCore/qset.h> -#include <qgst_p.h> -#include <gst/video/video.h> -#include <qaudioformat.h> -#include <qcamera.h> -#include <qvideoframe.h> -#include <QDebug> +#include <gst/gstsample.h> +#include <gst/gstbuffer.h> + +#include <QtCore/qglobal.h> QT_BEGIN_NAMESPACE -class QSize; -class QVariant; -class QByteArray; -class QImage; -class QVideoFrameFormat; +class QAudioFormat; +class QGstCaps; +class QVideoFrame; namespace QGstUtils { - Q_MULTIMEDIA_EXPORT QAudioFormat audioFormatForSample(GstSample *sample); - QAudioFormat audioFormatForCaps(QGstCaps caps); - Q_MULTIMEDIA_EXPORT QGstMutableCaps capsForAudioFormat(const QAudioFormat &format); +QAudioFormat audioFormatForSample(GstSample *sample); +QAudioFormat audioFormatForCaps(const QGstCaps &caps); +QGstCaps capsForAudioFormat(const QAudioFormat &format); - void setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer); -} +void setFrameTimeStampsFromBuffer(QVideoFrame *frame, GstBuffer *buffer); +} // namespace QGstUtils -Q_MULTIMEDIA_EXPORT GList *qt_gst_video_sinks(); +GList *qt_gst_video_sinks(); QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp index 0d22999b2..df3fb3d69 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstvideobuffer_p.h" #include "qgstreamervideosink_p.h" @@ -48,24 +12,24 @@ #include <gst/video/gstvideometa.h> #include <gst/pbutils/gstpluginsbaseversion.h> -#include "qgstutils_p.h" +#include <common/qgstutils_p.h> #if QT_CONFIG(gstreamer_gl) -#include <QtGui/private/qrhi_p.h> -#include <QtGui/private/qrhigles2_p.h> -#include <QtGui/qopenglcontext.h> -#include <QtGui/qopenglfunctions.h> -#include <QtGui/qopengl.h> - -#include <gst/gl/gstglconfig.h> -#include <gst/gl/gstglmemory.h> -#include <gst/gl/gstglsyncmeta.h> -#if QT_CONFIG(linux_dmabuf) -#include <gst/allocators/gstdmabuf.h> -#endif +# include <QtGui/rhi/qrhi.h> +# include <QtGui/qopenglcontext.h> +# include <QtGui/qopenglfunctions.h> +# include <QtGui/qopengl.h> + +# include <gst/gl/gstglconfig.h> +# include <gst/gl/gstglmemory.h> +# include <gst/gl/gstglsyncmeta.h> -#include <EGL/egl.h> -#include <EGL/eglext.h> +# include <EGL/egl.h> +# include <EGL/eglext.h> + +# if QT_CONFIG(linux_dmabuf) +# include <gst/allocators/gstdmabuf.h> +# endif #endif QT_BEGIN_NAMESPACE @@ -87,79 +51,60 @@ QT_BEGIN_NAMESPACE #define DRM_FORMAT_GR1616 fourcc_code('G', 'R', '3', '2') /* [31:0] G:R 16:16 little endian */ #define DRM_FORMAT_BGRA1010102 fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */ -QGstVideoBuffer::QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, - const QVideoFrameFormat &frameFormat, +QGstVideoBuffer::QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, + QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format) - : QAbstractVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) ? - QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, sink ? sink->rhi() : nullptr) - , memoryFormat(format) - , m_frameFormat(frameFormat) - , m_rhi(sink ? sink->rhi() : nullptr) - , m_videoInfo(info) - , m_buffer(buffer) + : QHwVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) + ? QVideoFrame::RhiTextureHandle + : QVideoFrame::NoHandle, + sink ? sink->rhi() : nullptr), + memoryFormat(format), + m_frameFormat(frameFormat), + m_rhi(sink ? sink->rhi() : nullptr), + m_videoInfo(info), + m_buffer(std::move(buffer)) { - gst_buffer_ref(m_buffer); if (sink) { eglDisplay = sink->eglDisplay(); eglImageTargetTexture2D = sink->eglImageTargetTexture2D(); } -} - -QGstVideoBuffer::~QGstVideoBuffer() -{ - unmap(); - gst_buffer_unref(m_buffer); - if (m_syncBuffer) - gst_buffer_unref(m_syncBuffer); - - if (m_ownTextures && glContext) { - int planes = 0; - for (planes = 0; planes < 3; ++planes) { - if (m_textures[planes] == 0) - break; - } -#if QT_CONFIG(gstreamer_gl) - if (m_rhi) { - m_rhi->makeThreadLocalNativeContextCurrent(); - QOpenGLFunctions functions(glContext); - functions.glDeleteTextures(planes, m_textures); - } +#if !QT_CONFIG(gstreamer_gl) + Q_UNUSED(memoryFormat); #endif - } } - -QVideoFrame::MapMode QGstVideoBuffer::mapMode() const +QGstVideoBuffer::~QGstVideoBuffer() { - return m_mode; + Q_ASSERT(m_mode == QtVideo::MapMode::NotMapped); } -QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) +QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QtVideo::MapMode mode) { - const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0) - | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0)); + const GstMapFlags flags = GstMapFlags( + ((mode & QtVideo::MapMode::ReadOnly ) == QtVideo::MapMode::NotMapped ? 0 : GST_MAP_READ) + | ((mode & QtVideo::MapMode::WriteOnly) == QtVideo::MapMode::NotMapped ? 0 : GST_MAP_WRITE)); MapData mapData; - if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped) + if (mode == QtVideo::MapMode::NotMapped || m_mode != QtVideo::MapMode::NotMapped) return mapData; if (m_videoInfo.finfo->n_planes == 0) { // Encoded - if (gst_buffer_map(m_buffer, &m_frame.map[0], flags)) { - mapData.nPlanes = 1; + if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) { + mapData.planeCount = 1; mapData.bytesPerLine[0] = -1; - mapData.size[0] = m_frame.map[0].size; + mapData.dataSize[0] = m_frame.map[0].size; mapData.data[0] = static_cast<uchar *>(m_frame.map[0].data); m_mode = mode; } - } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, flags)) { - mapData.nPlanes = GST_VIDEO_FRAME_N_PLANES(&m_frame); + } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer.get(), flags)) { + mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame); for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) { mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i); mapData.data[i] = static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i)); - mapData.size[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); + mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); } m_mode = mode; @@ -169,13 +114,13 @@ QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) void QGstVideoBuffer::unmap() { - if (m_mode != QVideoFrame::NotMapped) { + if (m_mode != QtVideo::MapMode::NotMapped) { if (m_videoInfo.finfo->n_planes == 0) - gst_buffer_unmap(m_buffer, &m_frame.map[0]); + gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]); else gst_video_frame_unmap(&m_frame); } - m_mode = QVideoFrame::NotMapped; + m_mode = QtVideo::MapMode::NotMapped; } #if QT_CONFIG(gstreamer_gl) && QT_CONFIG(linux_dmabuf) @@ -264,122 +209,185 @@ fourccFromVideoInfo(const GstVideoInfo * info, int plane) } #endif -void QGstVideoBuffer::mapTextures() +#if QT_CONFIG(gstreamer_gl) +struct GlTextures { - if (!m_rhi) - return; + uint count = 0; + bool owned = false; + std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{}; +}; -#if QT_CONFIG(gstreamer_gl) - if (memoryFormat == QGstCaps::GLTexture) { - auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(m_buffer, 0)); - Q_ASSERT(mem); - if (!gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) { - qWarning() << "Could not map GL textures"; - } else { - auto *sync_meta = gst_buffer_get_gl_sync_meta(m_buffer); - - if (!sync_meta) { - m_syncBuffer = gst_buffer_new(); - sync_meta = gst_buffer_add_gl_sync_meta(mem->context, m_syncBuffer); - } - gst_gl_sync_meta_set_sync_point (sync_meta, mem->context); - gst_gl_sync_meta_wait (sync_meta, mem->context); - - int nPlanes = m_frame.info.finfo->n_planes; - for (int i = 0; i < nPlanes; ++i) { - m_textures[i] = *(guint32 *)m_frame.data[i]; - } - gst_video_frame_unmap(&m_frame); +class QGstQVideoFrameTextures : public QVideoFrameTextures +{ +public: + QGstQVideoFrameTextures(QRhi *rhi, QSize size, QVideoFrameFormat::PixelFormat format, GlTextures &textures) + : m_rhi(rhi) + , m_glTextures(textures) + { + auto desc = QVideoTextureHelper::textureDescription(format); + for (uint i = 0; i < textures.count; ++i) { + QSize planeSize(desc->widthForPlane(size.width(), int(i)), + desc->heightForPlane(size.height(), int(i))); + m_textures[i].reset(rhi->newTexture(desc->textureFormat[i], planeSize, 1, {})); + m_textures[i]->createFrom({textures.names[i], 0}); } } + + ~QGstQVideoFrameTextures() + { + m_rhi->makeThreadLocalNativeContextCurrent(); + auto ctx = QOpenGLContext::currentContext(); + if (m_glTextures.owned && ctx) + ctx->functions()->glDeleteTextures(int(m_glTextures.count), m_glTextures.names.data()); + } + + QRhiTexture *texture(uint plane) const override + { + return plane < m_glTextures.count ? m_textures[plane].get() : nullptr; + } + +private: + QRhi *m_rhi = nullptr; + GlTextures m_glTextures; + std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes]; +}; + +static GlTextures mapFromGlTexture(const QGstBufferHandle &bufferHandle, GstVideoFrame &frame, + GstVideoInfo &videoInfo) +{ + GstBuffer *buffer = bufferHandle.get(); + auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0)); + if (!mem) + return {}; + + if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) { + qWarning() << "Could not map GL textures"; + return {}; + } + + auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer); + GstBuffer *sync_buffer = nullptr; + if (!sync_meta) { + sync_buffer = gst_buffer_new(); + sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer); + } + gst_gl_sync_meta_set_sync_point (sync_meta, mem->context); + gst_gl_sync_meta_wait (sync_meta, mem->context); + if (sync_buffer) + gst_buffer_unref(sync_buffer); + + GlTextures textures; + textures.count = frame.info.finfo->n_planes; + + for (uint i = 0; i < textures.count; ++i) + textures.names[i] = *(guint32 *)frame.data[i]; + + gst_video_frame_unmap(&frame); + + return textures; +} + #if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) - else if (memoryFormat == QGstCaps::DMABuf) { - if (m_textures[0]) - return; - Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(m_buffer, 0))); - Q_ASSERT(eglDisplay); - Q_ASSERT(eglImageTargetTexture2D); - - auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(m_rhi->nativeHandles()); - glContext = nativeHandles->context; - if (!glContext) { - qWarning() << "no GL context"; - return; - } +static GlTextures mapFromDmaBuffer(QRhi *rhi, const QGstBufferHandle &bufferHandle, + GstVideoFrame &frame, GstVideoInfo &videoInfo, + Qt::HANDLE eglDisplay, QFunctionPointer eglImageTargetTexture2D) +{ + GstBuffer *buffer = bufferHandle.get(); - if (!gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, GstMapFlags(GST_MAP_READ))) { - qDebug() << "Couldn't map DMA video frame"; - return; - } + Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0))); + Q_ASSERT(eglDisplay); + Q_ASSERT(eglImageTargetTexture2D); - int nPlanes = GST_VIDEO_FRAME_N_PLANES(&m_frame); -// int width = GST_VIDEO_FRAME_WIDTH(&m_frame); -// int height = GST_VIDEO_FRAME_HEIGHT(&m_frame); - Q_ASSERT(GST_VIDEO_FRAME_N_PLANES(&m_frame) == gst_buffer_n_memory(m_buffer)); - - QOpenGLFunctions functions(glContext); - functions.glGenTextures(nPlanes, m_textures); - m_ownTextures = true; - -// qDebug() << Qt::hex << "glGenTextures: glerror" << glGetError() << "egl error" << eglGetError(); -// qDebug() << "converting DMA buffer nPlanes=" << nPlanes << m_textures[0] << m_textures[1] << m_textures[2]; - - for (int i = 0; i < nPlanes; ++i) { - auto offset = GST_VIDEO_FRAME_PLANE_OFFSET(&m_frame, i); - auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i); - int planeWidth = GST_VIDEO_FRAME_COMP_WIDTH(&m_frame, i); - int planeHeight = GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); - auto mem = gst_buffer_peek_memory(m_buffer, i); - int fd = gst_dmabuf_memory_get_fd(mem); - -// qDebug() << " plane" << i << "size" << width << height << "stride" << stride << "offset" << offset << "fd=" << fd; - // ### do we need to open/close the fd? - // ### can we convert several planes at once? - // Get the correct DRM_FORMATs from the texture format in the description - EGLAttrib const attribute_list[] = { - EGL_WIDTH, planeWidth, - EGL_HEIGHT, planeHeight, - EGL_LINUX_DRM_FOURCC_EXT, fourccFromVideoInfo(&m_videoInfo, i), - EGL_DMA_BUF_PLANE0_FD_EXT, fd, - EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLAttrib)offset, - EGL_DMA_BUF_PLANE0_PITCH_EXT, stride, - EGL_NONE - }; - EGLImage image = eglCreateImage(eglDisplay, - EGL_NO_CONTEXT, - EGL_LINUX_DMA_BUF_EXT, - nullptr, - attribute_list); - if (image == EGL_NO_IMAGE_KHR) { - qWarning() << "could not create EGL image for plane" << i << Qt::hex << eglGetError(); - } -// qDebug() << Qt::hex << "eglCreateImage: glerror" << glGetError() << "egl error" << eglGetError(); - functions.glBindTexture(GL_TEXTURE_2D, m_textures[i]); -// qDebug() << Qt::hex << "bind texture: glerror" << glGetError() << "egl error" << eglGetError(); - auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D; - EGLImageTargetTexture2D(GL_TEXTURE_2D, image); -// qDebug() << Qt::hex << "glerror" << glGetError() << "egl error" << eglGetError(); - eglDestroyImage(eglDisplay, image); + auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); + auto glContext = nativeHandles->context; + if (!glContext) { + qWarning() << "no GL context"; + return {}; + } + + if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ))) { + qDebug() << "Couldn't map DMA video frame"; + return {}; + } + + GlTextures textures = {}; + textures.owned = true; + textures.count = GST_VIDEO_FRAME_N_PLANES(&frame); + // int width = GST_VIDEO_FRAME_WIDTH(&frame); + // int height = GST_VIDEO_FRAME_HEIGHT(&frame); + Q_ASSERT(GST_VIDEO_FRAME_N_PLANES(&frame) == gst_buffer_n_memory(buffer)); + + QOpenGLFunctions functions(glContext); + functions.glGenTextures(int(textures.count), textures.names.data()); + + // qDebug() << Qt::hex << "glGenTextures: glerror" << glGetError() << "egl error" << eglGetError(); + // qDebug() << "converting DMA buffer nPlanes=" << nPlanes << m_textures[0] << m_textures[1] << m_textures[2]; + + for (int i = 0; i < int(textures.count); ++i) { + auto offset = GST_VIDEO_FRAME_PLANE_OFFSET(&frame, i); + auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, i); + int planeWidth = GST_VIDEO_FRAME_COMP_WIDTH(&frame, i); + int planeHeight = GST_VIDEO_FRAME_COMP_HEIGHT(&frame, i); + auto mem = gst_buffer_peek_memory(buffer, i); + int fd = gst_dmabuf_memory_get_fd(mem); + + // qDebug() << " plane" << i << "size" << width << height << "stride" << stride << "offset" << offset << "fd=" << fd; + // ### do we need to open/close the fd? + // ### can we convert several planes at once? + // Get the correct DRM_FORMATs from the texture format in the description + EGLAttrib const attribute_list[] = { + EGL_WIDTH, planeWidth, + EGL_HEIGHT, planeHeight, + EGL_LINUX_DRM_FOURCC_EXT, fourccFromVideoInfo(&videoInfo, i), + EGL_DMA_BUF_PLANE0_FD_EXT, fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLAttrib)offset, + EGL_DMA_BUF_PLANE0_PITCH_EXT, stride, + EGL_NONE + }; + EGLImage image = eglCreateImage(eglDisplay, + EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, + attribute_list); + if (image == EGL_NO_IMAGE_KHR) { + qWarning() << "could not create EGL image for plane" << i << Qt::hex << eglGetError(); } - gst_video_frame_unmap(&m_frame); + // qDebug() << Qt::hex << "eglCreateImage: glerror" << glGetError() << "egl error" << eglGetError(); + functions.glBindTexture(GL_TEXTURE_2D, textures.names[i]); + // qDebug() << Qt::hex << "bind texture: glerror" << glGetError() << "egl error" << eglGetError(); + auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D; + EGLImageTargetTexture2D(GL_TEXTURE_2D, image); + // qDebug() << Qt::hex << "glerror" << glGetError() << "egl error" << eglGetError(); + eglDestroyImage(eglDisplay, image); } + gst_video_frame_unmap(&frame); + + return textures; +} #endif #endif - m_texturesUploaded = true; -} -std::unique_ptr<QRhiTexture> QGstVideoBuffer::texture(int plane) const +std::unique_ptr<QVideoFrameTextures> QGstVideoBuffer::mapTextures(QRhi *rhi) { - auto desc = QVideoTextureHelper::textureDescription(m_frameFormat.pixelFormat()); - if (!m_rhi || !desc || plane >= desc->nplanes) + if (!rhi) return {}; - QSize size(desc->widthForPlane(m_videoInfo.width, plane), desc->heightForPlane(m_videoInfo.height, plane)); - std::unique_ptr<QRhiTexture> tex(m_rhi->newTexture(desc->textureFormat[plane], size, 1, {})); - if (tex) { - if (!tex->createFrom({m_textures[plane], 0})) - return {}; - } - return tex; + +#if QT_CONFIG(gstreamer_gl) + GlTextures textures = {}; + if (memoryFormat == QGstCaps::GLTexture) + textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo); + +# if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) + else if (memoryFormat == QGstCaps::DMABuf) + textures = mapFromDmaBuffer(m_rhi, m_buffer, m_frame, m_videoInfo, eglDisplay, + eglImageTargetTexture2D); + +# endif + if (textures.count > 0) + return std::make_unique<QGstQVideoFrameTextures>(rhi, QSize{m_videoInfo.width, m_videoInfo.height}, + m_frameFormat.pixelFormat(), textures); +#endif + return {}; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h index 09dc8b992..573a4662c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTVIDEOBUFFER_P_H #define QGSTVIDEOBUFFER_P_H @@ -51,11 +15,10 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> #include <QtCore/qvariant.h> -#include <qgst_p.h> +#include <common/qgst_p.h> #include <gst/video/video.h> QT_BEGIN_NAMESPACE @@ -63,40 +26,28 @@ class QVideoFrameFormat; class QGstreamerVideoSink; class QOpenGLContext; -class Q_MULTIMEDIA_EXPORT QGstVideoBuffer : public QAbstractVideoBuffer +class QGstVideoBuffer final : public QHwVideoBuffer { public: - - QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, + QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format); - QGstVideoBuffer(GstBuffer *buffer, const QVideoFrameFormat &format, const GstVideoInfo &info) - : QGstVideoBuffer(buffer, info, nullptr, format, QGstCaps::CpuMemory) - {} ~QGstVideoBuffer(); - GstBuffer *buffer() const { return m_buffer; } - QVideoFrame::MapMode mapMode() const override; - - MapData map(QVideoFrame::MapMode mode) override; + MapData map(QtVideo::MapMode mode) override; void unmap() override; - void mapTextures() override; - std::unique_ptr<QRhiTexture> texture(int plane) const override; + std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *) override; + private: - QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; - QVideoFrameFormat m_frameFormat; + const QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; + const QVideoFrameFormat m_frameFormat; QRhi *m_rhi = nullptr; mutable GstVideoInfo m_videoInfo; - mutable GstVideoFrame m_frame; - GstBuffer *m_buffer = nullptr; - GstBuffer *m_syncBuffer = nullptr; - QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; - QOpenGLContext *glContext = nullptr; + mutable GstVideoFrame m_frame{}; + const QGstBufferHandle m_buffer; + QtVideo::MapMode m_mode = QtVideo::MapMode::NotMapped; Qt::HANDLE eglDisplay = nullptr; QFunctionPointer eglImageTargetTexture2D = nullptr; - uint m_textures[3] = {}; - bool m_texturesUploaded = false; - bool m_ownTextures = false; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp index 1a8107889..a2580973c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp @@ -1,64 +1,32 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qvideoframe.h> -#include <qvideosink.h> -#include <QDebug> -#include <QMap> -#include <QThread> -#include <QEvent> -#include <QCoreApplication> - -#include <private/qfactoryloader_p.h> -#include "qgstvideobuffer_p.h" -#include "qgstreamervideosink_p.h" +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstvideorenderersink_p.h" +#include <QtMultimedia/qvideoframe.h> +#include <QtMultimedia/qvideosink.h> +#include <QtCore/private/qfactoryloader_p.h> +#include <QtCore/private/quniquehandle_p.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qdebug.h> +#include <QtCore/qdebug.h> +#include <QtCore/qloggingcategory.h> +#include <QtCore/qmap.h> +#include <QtCore/qthread.h> +#include <QtGui/qevent.h> + +#include <common/qgstvideobuffer_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstutils_p.h> + +#include <private/qvideoframe_p.h> + #include <gst/video/video.h> #include <gst/video/gstvideometa.h> -#include <qloggingcategory.h> -#include <qdebug.h> -#include "qgstutils_p.h" -#include <QtGui/private/qrhi_p.h> +#include <rhi/qrhi.h> #if QT_CONFIG(gstreamer_gl) #include <gst/gl/gl.h> #endif // #if QT_CONFIG(gstreamer_gl) @@ -68,29 +36,27 @@ #include <gst/allocators/gstdmabuf.h> #endif -//#define DEBUG_VIDEO_SURFACE_SINK - -Q_LOGGING_CATEGORY(qLcGstVideoRenderer, "qt.multimedia.gstvideorenderer") +static Q_LOGGING_CATEGORY(qLcGstVideoRenderer, "qt.multimedia.gstvideorenderer") QT_BEGIN_NAMESPACE QGstVideoRenderer::QGstVideoRenderer(QGstreamerVideoSink *sink) - : m_sink(sink) + : m_sink(sink), m_surfaceCaps(createSurfaceCaps(sink)) { - createSurfaceCaps(); + QObject::connect( + sink, &QGstreamerVideoSink::aboutToBeDestroyed, this, + [this] { + QMutexLocker locker(&m_sinkMutex); + m_sink = nullptr; + }, + Qt::DirectConnection); } -QGstVideoRenderer::~QGstVideoRenderer() -{ -} +QGstVideoRenderer::~QGstVideoRenderer() = default; -void QGstVideoRenderer::createSurfaceCaps() +QGstCaps QGstVideoRenderer::createSurfaceCaps([[maybe_unused]] QGstreamerVideoSink *sink) { - QRhi *rhi = m_sink->rhi(); - Q_UNUSED(rhi); - - QGstMutableCaps caps; - caps.create(); + QGstCaps caps = QGstCaps::create(); // All the formats that both we and gstreamer support auto formats = QList<QVideoFrameFormat::PixelFormat>() @@ -115,10 +81,11 @@ void QGstVideoRenderer::createSurfaceCaps() << QVideoFrameFormat::Format_Y16 ; #if QT_CONFIG(gstreamer_gl) + QRhi *rhi = sink->rhi(); if (rhi && rhi->backend() == QRhi::OpenGLES2) { caps.addPixelFormats(formats, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); #if QT_CONFIG(linux_dmabuf) - if (m_sink->eglDisplay() && m_sink->eglImageTargetTexture2D()) { + if (sink->eglDisplay() && sink->eglImageTargetTexture2D()) { // We currently do not handle planar DMA buffers, as it's somewhat unclear how to // convert the planar EGLImage into something we can use from OpenGL auto singlePlaneFormats = QList<QVideoFrameFormat::PixelFormat>() @@ -142,105 +109,106 @@ void QGstVideoRenderer::createSurfaceCaps() } #endif caps.addPixelFormats(formats); - - m_surfaceCaps = caps; + return caps; } -QGstMutableCaps QGstVideoRenderer::caps() +const QGstCaps &QGstVideoRenderer::caps() { - QMutexLocker locker(&m_mutex); - return m_surfaceCaps; } -bool QGstVideoRenderer::start(GstCaps *caps) +bool QGstVideoRenderer::start(const QGstCaps& caps) { - qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << QGstCaps(caps).toString(); - QMutexLocker locker(&m_mutex); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << caps; - m_frameMirrored = false; - m_frameRotationAngle = QVideoFrame::Rotation0; - - if (m_active) { - m_flush = true; - m_stop = true; - } - - m_startCaps = QGstMutableCaps(caps, QGstMutableCaps::NeedsRef); - - /* - Waiting for start() to be invoked in the main thread may block - if gstreamer blocks the main thread until this call is finished. - This situation is rare and usually caused by setState(Null) - while pipeline is being prerolled. - - The proper solution to this involves controlling gstreamer pipeline from - other thread than video surface. - - Currently start() fails if wait() timed out. - */ - if (!waitForAsyncEvent(&locker, &m_setupCondition, 1000) && !m_startCaps.isNull()) { - qWarning() << "Failed to start video surface due to main thread blocked."; - m_startCaps = {}; + { + m_frameRotationAngle = QtVideo::Rotation::None; + auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo(); + if (optionalFormatAndVideoInfo) { + std::tie(m_format, m_videoInfo) = std::move(*optionalFormatAndVideoInfo); + } else { + m_format = {}; + m_videoInfo = {}; + } + m_memoryFormat = caps.memoryFormat(); } - return m_active; + return true; } void QGstVideoRenderer::stop() { - QMutexLocker locker(&m_mutex); - - if (!m_active) - return; - - m_flush = true; - m_stop = true; + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::stop"; - m_startCaps = {}; - - waitForAsyncEvent(&locker, &m_setupCondition, 500); + QMetaObject::invokeMethod(this, [this] { + m_currentState.buffer = {}; + m_currentPipelineFrame = {}; + updateCurrentVideoFrame(m_currentVideoFrame); + }); } void QGstVideoRenderer::unlock() { - QMutexLocker locker(&m_mutex); - - m_setupCondition.wakeAll(); - m_renderCondition.wakeAll(); -} - -bool QGstVideoRenderer::proposeAllocation(GstQuery *query) -{ - Q_UNUSED(query); - QMutexLocker locker(&m_mutex); - return m_active; + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::unlock"; } -void QGstVideoRenderer::flush() +bool QGstVideoRenderer::proposeAllocation(GstQuery *) { - QMutexLocker locker(&m_mutex); - - m_flush = true; - m_renderBuffer = nullptr; - m_renderCondition.wakeAll(); - - notify(); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation"; + return true; } GstFlowReturn QGstVideoRenderer::render(GstBuffer *buffer) { - QMutexLocker locker(&m_mutex); qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::render"; - m_renderReturn = GST_FLOW_OK; - m_renderBuffer = buffer; + GstVideoCropMeta *meta = gst_buffer_get_video_crop_meta(buffer); + if (meta) { + QRect vp(meta->x, meta->y, meta->width, meta->height); + if (m_format.viewport() != vp) { + qCDebug(qLcGstVideoRenderer) + << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x" + << meta->width << " | " << meta->x << "x" << meta->y << "]"; + // Update viewport if data is not the same + m_format.setViewport(vp); + } + } - waitForAsyncEvent(&locker, &m_renderCondition, 300); + RenderBufferState state{ + .buffer = QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef }, + .format = m_format, + .memoryFormat = m_memoryFormat, + .mirrored = m_frameMirrored, + .rotationAngle = m_frameRotationAngle, + }; + + qCDebug(qLcGstVideoRenderer) << " sending video frame"; + + QMetaObject::invokeMethod(this, [this, state = std::move(state)]() mutable { + if (state == m_currentState) + // same buffer received twice + return; + + auto videoBuffer = std::make_unique<QGstVideoBuffer>(state.buffer, m_videoInfo, m_sink, + state.format, state.memoryFormat); + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(videoBuffer), state.format); + QGstUtils::setFrameTimeStampsFromBuffer(&frame, state.buffer.get()); + frame.setMirrored(state.mirrored); + frame.setRotation(state.rotationAngle); + + m_currentPipelineFrame = std::move(frame); + m_currentState = std::move(state); + + if (!m_isActive) { + qCDebug(qLcGstVideoRenderer) << " showing empty video frame"; + updateCurrentVideoFrame({}); + return; + } - m_renderBuffer = nullptr; + updateCurrentVideoFrame(m_currentPipelineFrame); + }); - return m_renderReturn; + return GST_FLOW_OK; } bool QGstVideoRenderer::query(GstQuery *query) @@ -253,6 +221,10 @@ bool QGstVideoRenderer::query(GstQuery *query) if (strcmp(type, "gst.gl.local_context") != 0) return false; + QMutexLocker locker(&m_sinkMutex); + if (!m_sink) + return false; + auto *gstGlContext = m_sink->gstGlLocalContext(); if (!gstGlContext) return false; @@ -269,15 +241,47 @@ bool QGstVideoRenderer::query(GstQuery *query) void QGstVideoRenderer::gstEvent(GstEvent *event) { - if (GST_EVENT_TYPE(event) != GST_EVENT_TAG) + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_TAG: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: Tag"; + return gstEventHandleTag(event); + case GST_EVENT_EOS: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: EOS"; + return gstEventHandleEOS(event); + + default: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: unhandled event - " << event; + return; + } +} + +void QGstVideoRenderer::setActive(bool isActive) +{ + if (isActive == m_isActive) return; + m_isActive = isActive; + if (isActive) + updateCurrentVideoFrame(m_currentPipelineFrame); + else + updateCurrentVideoFrame({}); +} + +void QGstVideoRenderer::updateCurrentVideoFrame(QVideoFrame frame) +{ + m_currentVideoFrame = std::move(frame); + if (m_sink) + m_sink->setVideoFrame(m_currentVideoFrame); +} + +void QGstVideoRenderer::gstEventHandleTag(GstEvent *event) +{ GstTagList *taglist = nullptr; gst_event_parse_tag(event, &taglist); if (!taglist) return; - gchar *value = nullptr; + QGString value; if (!gst_tag_list_get_string(taglist, GST_TAG_IMAGE_ORIENTATION, &value)) return; @@ -289,203 +293,79 @@ void QGstVideoRenderer::gstEvent(GstEvent *event) bool mirrored = false; int rotationAngle = 0; - if (!strncmp(rotate, value, rotateLen)) { - rotationAngle = atoi(value + rotateLen); - } else if (!strncmp(flipRotate, value, flipRotateLen)) { + if (!strncmp(rotate, value.get(), rotateLen)) { + rotationAngle = atoi(value.get() + rotateLen); + } else if (!strncmp(flipRotate, value.get(), flipRotateLen)) { // To flip by horizontal axis is the same as to mirror by vertical axis // and rotate by 180 degrees. mirrored = true; - rotationAngle = (180 + atoi(value + flipRotateLen)) % 360; + rotationAngle = (180 + atoi(value.get() + flipRotateLen)) % 360; } - QMutexLocker locker(&m_mutex); m_frameMirrored = mirrored; switch (rotationAngle) { - case 0: m_frameRotationAngle = QVideoFrame::Rotation0; break; - case 90: m_frameRotationAngle = QVideoFrame::Rotation90; break; - case 180: m_frameRotationAngle = QVideoFrame::Rotation180; break; - case 270: m_frameRotationAngle = QVideoFrame::Rotation270; break; - default: m_frameRotationAngle = QVideoFrame::Rotation0; + case 0: + m_frameRotationAngle = QtVideo::Rotation::None; + break; + case 90: + m_frameRotationAngle = QtVideo::Rotation::Clockwise90; + break; + case 180: + m_frameRotationAngle = QtVideo::Rotation::Clockwise180; + break; + case 270: + m_frameRotationAngle = QtVideo::Rotation::Clockwise270; + break; + default: + m_frameRotationAngle = QtVideo::Rotation::None; } } -bool QGstVideoRenderer::event(QEvent *event) +void QGstVideoRenderer::gstEventHandleEOS(GstEvent *) { - if (event->type() == QEvent::UpdateRequest) { - QMutexLocker locker(&m_mutex); - - if (m_notified) { - while (handleEvent(&locker)) {} - m_notified = false; - } - return true; - } - - return QObject::event(event); + stop(); } -bool QGstVideoRenderer::handleEvent(QMutexLocker<QMutex> *locker) -{ - if (m_flush) { - m_flush = false; - if (m_active) { - locker->unlock(); - - if (m_sink && !m_flushed) - m_sink->setVideoFrame(QVideoFrame()); - m_flushed = true; - } - } else if (m_stop) { - m_stop = false; - - if (m_active) { - m_active = false; - m_flushed = true; - } - } else if (!m_startCaps.isNull()) { - Q_ASSERT(!m_active); - - auto startCaps = m_startCaps; - m_startCaps = nullptr; - - if (m_sink) { - locker->unlock(); - - m_flushed = true; - m_format = startCaps.formatForCaps(&m_videoInfo); - memoryFormat = startCaps.memoryFormat(); - - locker->relock(); - m_active = m_format.isValid(); - } else if (m_active) { - m_active = false; - m_flushed = true; - } - - } else if (m_renderBuffer) { - GstBuffer *buffer = m_renderBuffer; - m_renderBuffer = nullptr; - m_renderReturn = GST_FLOW_ERROR; - - qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::handleEvent(renderBuffer)" << m_active << m_sink; - if (m_active && m_sink) { - gst_buffer_ref(buffer); - - locker->unlock(); - - m_flushed = false; - - auto meta = gst_buffer_get_video_crop_meta (buffer); - if (meta) { - QRect vp(meta->x, meta->y, meta->width, meta->height); - if (m_format.viewport() != vp) { - qCDebug(qLcGstVideoRenderer) << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x" << meta->width << " | " << meta->x << "x" << meta->y << "]"; - // Update viewport if data is not the same - m_format.setViewport(vp); - } - } - - if (m_sink->inStoppedState()) { - qCDebug(qLcGstVideoRenderer) << " sending empty video frame"; - m_sink->setVideoFrame(QVideoFrame()); - } else { - QGstVideoBuffer *videoBuffer = new QGstVideoBuffer(buffer, m_videoInfo, m_sink, m_format, memoryFormat); - QVideoFrame frame(videoBuffer, m_format); - QGstUtils::setFrameTimeStamps(&frame, buffer); - frame.setMirrored(m_frameMirrored); - frame.setRotationAngle(m_frameRotationAngle); - - qCDebug(qLcGstVideoRenderer) << " sending video frame"; - m_sink->setVideoFrame(frame); - } - - gst_buffer_unref(buffer); - - locker->relock(); - - m_renderReturn = GST_FLOW_OK; - } - - m_renderCondition.wakeAll(); - } else { - m_setupCondition.wakeAll(); - - return false; - } - return true; -} - -void QGstVideoRenderer::notify() -{ - if (!m_notified) { - m_notified = true; - QCoreApplication::postEvent(this, new QEvent(QEvent::UpdateRequest)); - } -} - -bool QGstVideoRenderer::waitForAsyncEvent( - QMutexLocker<QMutex> *locker, QWaitCondition *condition, unsigned long time) -{ - if (QThread::currentThread() == thread()) { - while (handleEvent(locker)) {} - m_notified = false; - - return true; - } - - notify(); - - return condition->wait(&m_mutex, time); -} - -static GstVideoSinkClass *sink_parent_class; -static thread_local QGstreamerVideoSink *current_sink; +static GstVideoSinkClass *gvrs_sink_parent_class; +static thread_local QGstreamerVideoSink *gvrs_current_sink; #define VO_SINK(s) QGstVideoRendererSink *sink(reinterpret_cast<QGstVideoRendererSink *>(s)) -QGstVideoRendererSink *QGstVideoRendererSink::createSink(QGstreamerVideoSink *sink) +QGstVideoRendererSinkElement QGstVideoRendererSink::createSink(QGstreamerVideoSink *sink) { setSink(sink); QGstVideoRendererSink *gstSink = reinterpret_cast<QGstVideoRendererSink *>( g_object_new(QGstVideoRendererSink::get_type(), nullptr)); - g_signal_connect(G_OBJECT(gstSink), "notify::show-preroll-frame", G_CALLBACK(handleShowPrerollChange), gstSink); - - return gstSink; + return QGstVideoRendererSinkElement{ + gstSink, + QGstElement::NeedsRef, + }; } void QGstVideoRendererSink::setSink(QGstreamerVideoSink *sink) { - current_sink = sink; - get_type(); + gvrs_current_sink = sink; } GType QGstVideoRendererSink::get_type() { - static GType type = 0; - - if (type == 0) { - static const GTypeInfo info = - { - sizeof(QGstVideoRendererSinkClass), // class_size - base_init, // base_init - nullptr, // base_finalize - class_init, // class_init - nullptr, // class_finalize - nullptr, // class_data - sizeof(QGstVideoRendererSink), // instance_size - 0, // n_preallocs - instance_init, // instance_init - nullptr // value_table - }; - - type = g_type_register_static( - GST_TYPE_VIDEO_SINK, "QGstVideoRendererSink", &info, GTypeFlags(0)); - - // Register the sink type to be used in custom piplines. - // When surface is ready the sink can be used. - gst_element_register(nullptr, "qtvideosink", GST_RANK_PRIMARY, type); - } + static const GTypeInfo info = + { + sizeof(QGstVideoRendererSinkClass), // class_size + base_init, // base_init + nullptr, // base_finalize + class_init, // class_init + nullptr, // class_finalize + nullptr, // class_data + sizeof(QGstVideoRendererSink), // instance_size + 0, // n_preallocs + instance_init, // instance_init + nullptr // value_table + }; + + static const GType type = g_type_register_static(GST_TYPE_VIDEO_SINK, "QGstVideoRendererSink", + &info, GTypeFlags(0)); return type; } @@ -494,7 +374,7 @@ void QGstVideoRendererSink::class_init(gpointer g_class, gpointer class_data) { Q_UNUSED(class_data); - sink_parent_class = reinterpret_cast<GstVideoSinkClass *>(g_type_class_peek_parent(g_class)); + gvrs_sink_parent_class = reinterpret_cast<GstVideoSinkClass *>(g_type_class_peek_parent(g_class)); GstVideoSinkClass *video_sink_class = reinterpret_cast<GstVideoSinkClass *>(g_class); video_sink_class->show_frame = QGstVideoRendererSink::show_frame; @@ -538,11 +418,11 @@ void QGstVideoRendererSink::instance_init(GTypeInstance *instance, gpointer g_cl Q_UNUSED(g_class); VO_SINK(instance); - Q_ASSERT(current_sink); + Q_ASSERT(gvrs_current_sink); - sink->renderer = new QGstVideoRenderer(current_sink); - sink->renderer->moveToThread(current_sink->thread()); - current_sink = nullptr; + sink->renderer = new QGstVideoRenderer(gvrs_current_sink); + sink->renderer->moveToThread(gvrs_current_sink->thread()); + gvrs_current_sink = nullptr; } void QGstVideoRendererSink::finalize(GObject *object) @@ -552,74 +432,39 @@ void QGstVideoRendererSink::finalize(GObject *object) delete sink->renderer; // Chain up - G_OBJECT_CLASS(sink_parent_class)->finalize(object); -} - -void QGstVideoRendererSink::handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d) -{ - Q_UNUSED(o); - Q_UNUSED(p); - QGstVideoRendererSink *sink = reinterpret_cast<QGstVideoRendererSink *>(d); - - gboolean showPrerollFrame = true; // "show-preroll-frame" property is true by default - g_object_get(G_OBJECT(sink), "show-preroll-frame", &showPrerollFrame, nullptr); - - if (!showPrerollFrame) { - GstState state = GST_STATE_VOID_PENDING; - GstClockTime timeout = 10000000; // 10 ms - gst_element_get_state(GST_ELEMENT(sink), &state, nullptr, timeout); - // show-preroll-frame being set to 'false' while in GST_STATE_PAUSED means - // the QMediaPlayer was stopped from the paused state. - // We need to flush the current frame. - if (state == GST_STATE_PAUSED) - sink->renderer->flush(); - } + G_OBJECT_CLASS(gvrs_sink_parent_class)->finalize(object); } GstStateChangeReturn QGstVideoRendererSink::change_state( GstElement *element, GstStateChange transition) { - QGstVideoRendererSink *sink = reinterpret_cast<QGstVideoRendererSink *>(element); - - gboolean showPrerollFrame = true; // "show-preroll-frame" property is true by default - g_object_get(G_OBJECT(element), "show-preroll-frame", &showPrerollFrame, nullptr); - - // If show-preroll-frame is 'false' when transitioning from GST_STATE_PLAYING to - // GST_STATE_PAUSED, it means the QMediaPlayer was stopped. - // We need to flush the current frame. - if (transition == GST_STATE_CHANGE_PLAYING_TO_PAUSED && !showPrerollFrame) - sink->renderer->flush(); - - return GST_ELEMENT_CLASS(sink_parent_class)->change_state(element, transition); + return GST_ELEMENT_CLASS(gvrs_sink_parent_class)->change_state(element, transition); } GstCaps *QGstVideoRendererSink::get_caps(GstBaseSink *base, GstCaps *filter) { VO_SINK(base); - QGstMutableCaps caps = sink->renderer->caps(); + QGstCaps caps = sink->renderer->caps(); if (filter) - caps = gst_caps_intersect(caps.get(), filter); + caps = QGstCaps(gst_caps_intersect(caps.caps(), filter), QGstCaps::HasRef); - gst_caps_ref(caps.get()); - return caps.get(); + return caps.release(); } -gboolean QGstVideoRendererSink::set_caps(GstBaseSink *base, GstCaps *caps) +gboolean QGstVideoRendererSink::set_caps(GstBaseSink *base, GstCaps *gcaps) { VO_SINK(base); + auto caps = QGstCaps(gcaps, QGstCaps::NeedsRef); - qCDebug(qLcGstVideoRenderer) << "set_caps:" << QGstCaps(caps).toString(); + qCDebug(qLcGstVideoRenderer) << "set_caps:" << caps; - if (!caps) { + if (caps.isNull()) { sink->renderer->stop(); - return TRUE; - } else if (sink->renderer->start(caps)) { - return TRUE; - } else { - return FALSE; } + + return sink->renderer->start(caps); } gboolean QGstVideoRendererSink::propose_allocation(GstBaseSink *base, GstQuery *query) @@ -654,14 +499,33 @@ gboolean QGstVideoRendererSink::query(GstBaseSink *base, GstQuery *query) if (sink->renderer->query(query)) return TRUE; - return GST_BASE_SINK_CLASS(sink_parent_class)->query(base, query); + return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->query(base, query); } gboolean QGstVideoRendererSink::event(GstBaseSink *base, GstEvent * event) { VO_SINK(base); sink->renderer->gstEvent(event); - return GST_BASE_SINK_CLASS(sink_parent_class)->event(base, event); + return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->event(base, event); +} + +QGstVideoRendererSinkElement::QGstVideoRendererSinkElement(QGstVideoRendererSink *element, + RefMode mode) + : QGstBaseSink{ + qGstCheckedCast<GstBaseSink>(element), + mode, + } +{ +} + +void QGstVideoRendererSinkElement::setActive(bool isActive) +{ + qGstVideoRendererSink()->renderer->setActive(isActive); +} + +QGstVideoRendererSink *QGstVideoRendererSinkElement::qGstVideoRendererSink() const +{ + return reinterpret_cast<QGstVideoRendererSink *>(element()); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h index 18530f62e..60cd25db0 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTVIDEORENDERERSINK_P_H #define QGSTVIDEORENDERERSINK_P_H @@ -51,7 +15,11 @@ // We mean it. // +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/qvideoframe.h> #include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtCore/qmutex.h> + #include <gst/video/gstvideosink.h> #include <gst/video/video.h> @@ -62,79 +30,85 @@ #include <QtCore/qwaitcondition.h> #include <qvideoframeformat.h> #include <qvideoframe.h> -#include <qgstvideobuffer_p.h> -#include <qgst_p.h> +#include <common/qgstvideobuffer_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QVideoSink; class QGstVideoRenderer : public QObject { - Q_OBJECT public: - QGstVideoRenderer(QGstreamerVideoSink *sink); + explicit QGstVideoRenderer(QGstreamerVideoSink *); ~QGstVideoRenderer(); - QGstMutableCaps caps(); + const QGstCaps &caps(); - bool start(GstCaps *caps); + bool start(const QGstCaps &); void stop(); void unlock(); - bool proposeAllocation(GstQuery *query); - - void flush(); - - GstFlowReturn render(GstBuffer *buffer); + bool proposeAllocation(GstQuery *); + GstFlowReturn render(GstBuffer *); + bool query(GstQuery *); + void gstEvent(GstEvent *); - bool event(QEvent *event) override; - bool query(GstQuery *query); - void gstEvent(GstEvent *event); - -private slots: - bool handleEvent(QMutexLocker<QMutex> *locker); + void setActive(bool); private: - void notify(); - bool waitForAsyncEvent(QMutexLocker<QMutex> *locker, QWaitCondition *condition, unsigned long time); - void createSurfaceCaps(); - - QPointer<QGstreamerVideoSink> m_sink; + void updateCurrentVideoFrame(QVideoFrame); - QMutex m_mutex; - QWaitCondition m_setupCondition; - QWaitCondition m_renderCondition; - - // --- accessed from multiple threads, need to hold mutex to access - GstFlowReturn m_renderReturn = GST_FLOW_OK; - bool m_active = false; - - QGstMutableCaps m_surfaceCaps; + void notify(); + static QGstCaps createSurfaceCaps(QGstreamerVideoSink *); - QGstMutableCaps m_startCaps; - GstBuffer *m_renderBuffer = nullptr; + void gstEventHandleTag(GstEvent *); + void gstEventHandleEOS(GstEvent *); - bool m_notified = false; - bool m_stop = false; - bool m_flush = false; - bool m_frameMirrored = false; - QVideoFrame::RotationAngle m_frameRotationAngle = QVideoFrame::Rotation0; + QMutex m_sinkMutex; + QGstreamerVideoSink *m_sink = nullptr; // written only from qt thread. so only readers on + // worker threads need to acquire the lock - // --- only accessed from one thread + // --- only accessed from gstreamer thread + const QGstCaps m_surfaceCaps; QVideoFrameFormat m_format; - GstVideoInfo m_videoInfo; - bool m_flushed = true; - QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; + GstVideoInfo m_videoInfo{}; + QGstCaps::MemoryFormat m_memoryFormat = QGstCaps::CpuMemory; + bool m_frameMirrored = false; + QtVideo::Rotation m_frameRotationAngle = QtVideo::Rotation::None; + + // --- only accessed from qt thread + QVideoFrame m_currentPipelineFrame; + QVideoFrame m_currentVideoFrame; + bool m_isActive{ false }; + + struct RenderBufferState + { + QGstBufferHandle buffer; + QVideoFrameFormat format; + QGstCaps::MemoryFormat memoryFormat; + bool mirrored; + QtVideo::Rotation rotationAngle; + + bool operator==(const RenderBufferState &rhs) const + { + return std::tie(buffer, format, memoryFormat, mirrored, rotationAngle) + == std::tie(rhs.buffer, rhs.format, rhs.memoryFormat, rhs.mirrored, + rhs.rotationAngle); + } + }; + RenderBufferState m_currentState; }; -class Q_MULTIMEDIA_EXPORT QGstVideoRendererSink +class QGstVideoRendererSinkElement; + +class QGstVideoRendererSink { public: - GstVideoSink parent; + GstVideoSink parent{}; - static QGstVideoRendererSink *createSink(QGstreamerVideoSink *surface); - static void setSink(QGstreamerVideoSink *surface); + static QGstVideoRendererSinkElement createSink(QGstreamerVideoSink *surface); private: + static void setSink(QGstreamerVideoSink *surface); + static GType get_type(); static void class_init(gpointer g_class, gpointer class_data); static void base_init(gpointer g_class); @@ -142,8 +116,6 @@ private: static void finalize(GObject *object); - static void handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d); - static GstStateChangeReturn change_state(GstElement *element, GstStateChange transition); static GstCaps *get_caps(GstBaseSink *sink, GstCaps *filter); @@ -157,19 +129,36 @@ private: static GstFlowReturn show_frame(GstVideoSink *sink, GstBuffer *buffer); static gboolean query(GstBaseSink *element, GstQuery *query); - static gboolean event(GstBaseSink *element, GstEvent * event); + static gboolean event(GstBaseSink *element, GstEvent *event); + + friend class QGstVideoRendererSinkElement; -private: QGstVideoRenderer *renderer = nullptr; }; - class QGstVideoRendererSinkClass { public: GstVideoSinkClass parent_class; }; +class QGstVideoRendererSinkElement : public QGstBaseSink +{ +public: + using QGstBaseSink::QGstBaseSink; + + explicit QGstVideoRendererSinkElement(QGstVideoRendererSink *, RefMode); + + QGstVideoRendererSinkElement(const QGstVideoRendererSinkElement &) = default; + QGstVideoRendererSinkElement(QGstVideoRendererSinkElement &&) noexcept = default; + QGstVideoRendererSinkElement &operator=(const QGstVideoRendererSinkElement &) = default; + QGstVideoRendererSinkElement &operator=(QGstVideoRendererSinkElement &&) noexcept = default; + + void setActive(bool); + + QGstVideoRendererSink *qGstVideoRendererSink() const; +}; + QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp index 24f4301ee..c54e8b74b 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp @@ -1,78 +1,63 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qcameradevice.h> - -#include "qgstreamercamera_p.h" -#include "qgstreamerimagecapture_p.h" +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <mediacapture/qgstreamercamera_p.h> + +#include <QtMultimedia/qcameradevice.h> +#include <QtMultimedia/qmediacapturesession.h> +#include <QtMultimedia/private/qcameradevice_p.h> +#include <QtCore/qdebug.h> + +#include <common/qgst_debug_p.h> #include <qgstreamervideodevices_p.h> #include <qgstreamerintegration_p.h> -#include <qmediacapturesession.h> #if QT_CONFIG(linux_v4l) #include <linux/videodev2.h> #include <private/qcore_unix_p.h> #endif -#include <QtCore/qdebug.h> + +QT_BEGIN_NAMESPACE + +QMaybe<QPlatformCamera *> QGstreamerCamera::create(QCamera *camera) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "videotestsrc", "capsfilter", "videoconvert", "videoscale", "identity"); + if (error) + return *error; + + return new QGstreamerCamera(camera); +} QGstreamerCamera::QGstreamerCamera(QCamera *camera) - : QPlatformCamera(camera) -{ - gstCamera = QGstElement("videotestsrc"); - gstCapsFilter = QGstElement("capsfilter", "videoCapsFilter"); - gstDecode = QGstElement("identity"); - gstVideoConvert = QGstElement("videoconvert", "videoConvert"); - gstVideoScale = QGstElement("videoscale", "videoScale"); - gstCameraBin = QGstBin("camerabin"); + : QGstreamerCameraBase(camera), + gstCameraBin{ + QGstBin::create("camerabin"), + }, + gstCamera{ + QGstElement::createFromFactory("videotestsrc"), + }, + gstCapsFilter{ + QGstElement::createFromFactory("capsfilter", "videoCapsFilter"), + }, + gstDecode{ + QGstElement::createFromFactory("identity"), + }, + gstVideoConvert{ + QGstElement::createFromFactory("videoconvert", "videoConvert"), + }, + gstVideoScale{ + QGstElement::createFromFactory("videoscale", "videoScale"), + } +{ gstCameraBin.add(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert, gstVideoScale); - gstCamera.link(gstCapsFilter, gstDecode, gstVideoConvert, gstVideoScale); + qLinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert, gstVideoScale); gstCameraBin.addGhostPad(gstVideoScale, "src"); } QGstreamerCamera::~QGstreamerCamera() { -#if QT_CONFIG(linux_v4l) - if (v4l2FileDescriptor >= 0) - qt_safe_close(v4l2FileDescriptor); - v4l2FileDescriptor = -1; -#endif gstCameraBin.setStateSync(GST_STATE_NULL); } @@ -95,6 +80,8 @@ void QGstreamerCamera::setActive(bool active) void QGstreamerCamera::setCamera(const QCameraDevice &camera) { + using namespace Qt::Literals; + if (m_cameraDevice == camera) return; @@ -102,48 +89,51 @@ void QGstreamerCamera::setCamera(const QCameraDevice &camera) QGstElement gstNewCamera; if (camera.isNull()) { - gstNewCamera = QGstElement("videotestsrc"); + gstNewCamera = QGstElement::createFromFactory("videotestsrc"); } else { auto *integration = static_cast<QGstreamerIntegration *>(QGstreamerIntegration::instance()); - auto *device = integration->videoDevice(camera.id()); - gstNewCamera = gst_device_create_element(device, "camerasrc"); - QGstStructure properties = gst_device_get_properties(device); - if (properties.name() == "v4l2deviceprovider") - m_v4l2Device = QString::fromUtf8(properties["device.path"].toString()); - } + GstDevice *device = integration->videoDevice(camera.id()); - QCameraFormat f = findBestCameraFormat(camera); - auto caps = QGstMutableCaps::fromCameraFormat(f); - auto gstNewDecode = QGstElement(f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); + if (!device) { + updateError(QCamera::Error::CameraError, + u"Failed to create GstDevice for camera: "_s + + QString::fromUtf8(camera.id())); + return; + } - gstCamera.unlink(gstCapsFilter); - gstCapsFilter.unlink(gstDecode); - gstDecode.unlink(gstVideoConvert); + gstNewCamera = QGstElement::createFromDevice(device, "camerasrc"); + QUniqueGstStructureHandle properties{ + gst_device_get_properties(device), + }; - gstCameraBin.remove(gstCamera); - gstCameraBin.remove(gstDecode); + if (properties) { + QGstStructureView propertiesView{ properties }; + if (propertiesView.name() == "v4l2deviceprovider") + m_v4l2DevicePath = QString::fromUtf8(propertiesView["device.path"].toString()); + } + } - gstCamera.setStateSync(GST_STATE_NULL); - gstDecode.setStateSync(GST_STATE_NULL); + QCameraFormat f = findBestCameraFormat(camera); + auto caps = QGstCaps::fromCameraFormat(f); + auto gstNewDecode = QGstElement::createFromFactory( + f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); - gstCapsFilter.set("caps", caps); + QGstPipeline::modifyPipelineWhileNotRunning(gstCamera.getPipeline(), [&] { + gstCamera.setStateSync(GST_STATE_READY); // stop camera, as it may have active tasks - gstCameraBin.add(gstNewCamera, gstNewDecode); + qUnlinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); + gstCameraBin.stopAndRemoveElements(gstCamera, gstDecode); - gstNewDecode.link(gstVideoConvert); - gstCapsFilter.link(gstNewDecode); + gstCapsFilter.set("caps", caps); - if (!gstNewCamera.link(gstCapsFilter)) - qWarning() << "linking camera failed" << gstCamera.name() << caps.toString(); + gstCamera = std::move(gstNewCamera); + gstDecode = std::move(gstNewDecode); - // Start sending frames once pipeline is linked - // FIXME: put camera to READY state before linking to decoder as in the NULL state it does not know its true caps - gstCapsFilter.syncStateWithParent(); - gstNewDecode.syncStateWithParent(); - gstNewCamera.syncStateWithParent(); + gstCameraBin.add(gstCamera, gstDecode); + qLinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); - gstCamera = gstNewCamera; - gstDecode = gstNewDecode; + gstCameraBin.syncChildrenState(); + }); updateCameraProperties(); } @@ -157,29 +147,25 @@ bool QGstreamerCamera::setCameraFormat(const QCameraFormat &format) if (f.isNull()) f = findBestCameraFormat(m_cameraDevice); - auto caps = QGstMutableCaps::fromCameraFormat(f); + auto caps = QGstCaps::fromCameraFormat(f); - auto newGstDecode = QGstElement(f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); - gstCameraBin.add(newGstDecode); - newGstDecode.syncStateWithParent(); + auto newGstDecode = QGstElement::createFromFactory( + f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); - gstCamera.staticPad("src").doInIdleProbe([&](){ - gstCamera.unlink(gstCapsFilter); - gstCapsFilter.unlink(gstDecode); - gstDecode.unlink(gstVideoConvert); + QGstPipeline::modifyPipelineWhileNotRunning(gstCamera.getPipeline(), [&] { + gstCamera.setStateSync(GST_STATE_READY); // stop camera, as it may have active tasks - gstCapsFilter.set("caps", caps); + qUnlinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); + gstCameraBin.stopAndRemoveElements(gstDecode); - newGstDecode.link(gstVideoConvert); - gstCapsFilter.link(newGstDecode); - if (!gstCamera.link(gstCapsFilter)) - qWarning() << "linking filtered camera to decoder failed" << gstCamera.name() << caps.toString(); - }); + gstCapsFilter.set("caps", caps); - gstCameraBin.remove(gstDecode); - gstDecode.setStateSync(GST_STATE_NULL); + gstDecode = std::move(newGstDecode); - gstDecode = newGstDecode; + gstCameraBin.add(gstDecode); + qLinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); + gstCameraBin.syncChildrenState(); + }); return true; } @@ -618,83 +604,78 @@ void QGstreamerCamera::setColorTemperature(int temperature) } #if QT_CONFIG(linux_v4l) +bool QGstreamerCamera::isV4L2Camera() const +{ + return !m_v4l2DevicePath.isEmpty(); +} + void QGstreamerCamera::initV4L2Controls() { v4l2AutoWhiteBalanceSupported = false; v4l2ColorTemperatureSupported = false; - QCamera::Features features; + QCamera::Features features{}; + Q_ASSERT(!m_v4l2DevicePath.isEmpty()); - const QString deviceName = v4l2Device(); - Q_ASSERT(!deviceName.isEmpty()); - v4l2FileDescriptor = qt_safe_open(deviceName.toLocal8Bit().constData(), O_RDONLY); - if (v4l2FileDescriptor == -1) { - qWarning() << "Unable to open the camera" << deviceName - << "for read to query the parameter info:" << qt_error_string(errno); - return; - } - - struct v4l2_queryctrl queryControl; - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_AUTO_WHITE_BALANCE; + withV4L2DeviceFileDescriptor([&](int fd) { + struct v4l2_queryctrl queryControl = {}; + queryControl.id = V4L2_CID_AUTO_WHITE_BALANCE; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2AutoWhiteBalanceSupported = true; - setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, true); - } + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2AutoWhiteBalanceSupported = true; + setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, true); + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_WHITE_BALANCE_TEMPERATURE; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinColorTemp = queryControl.minimum; - v4l2MaxColorTemp = queryControl.maximum; - v4l2ColorTemperatureSupported = true; - features |= QCamera::Feature::ColorTemperature; - } + queryControl = {}; + queryControl.id = V4L2_CID_WHITE_BALANCE_TEMPERATURE; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2MinColorTemp = queryControl.minimum; + v4l2MaxColorTemp = queryControl.maximum; + v4l2ColorTemperatureSupported = true; + features |= QCamera::Feature::ColorTemperature; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_EXPOSURE_AUTO; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2AutoExposureSupported = true; - } + queryControl = {}; + queryControl.id = V4L2_CID_EXPOSURE_AUTO; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2AutoExposureSupported = true; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_EXPOSURE_ABSOLUTE; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2ManualExposureSupported = true; - v4l2MinExposure = queryControl.minimum; - v4l2MaxExposure = queryControl.maximum; - features |= QCamera::Feature::ManualExposureTime; - } + queryControl = {}; + queryControl.id = V4L2_CID_EXPOSURE_ABSOLUTE; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2ManualExposureSupported = true; + v4l2MinExposure = queryControl.minimum; + v4l2MaxExposure = queryControl.maximum; + features |= QCamera::Feature::ManualExposureTime; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_AUTO_EXPOSURE_BIAS; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinExposureAdjustment = queryControl.minimum; - v4l2MaxExposureAdjustment = queryControl.maximum; - features |= QCamera::Feature::ExposureCompensation; - } + queryControl = {}; + queryControl.id = V4L2_CID_AUTO_EXPOSURE_BIAS; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2MinExposureAdjustment = queryControl.minimum; + v4l2MaxExposureAdjustment = queryControl.maximum; + features |= QCamera::Feature::ExposureCompensation; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_ISO_SENSITIVITY_AUTO; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - queryControl.id = V4L2_CID_ISO_SENSITIVITY; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - features |= QCamera::Feature::IsoSensitivity; - minIsoChanged(queryControl.minimum); - maxIsoChanged(queryControl.minimum); + queryControl = {}; + queryControl.id = V4L2_CID_ISO_SENSITIVITY_AUTO; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + queryControl.id = V4L2_CID_ISO_SENSITIVITY; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + features |= QCamera::Feature::IsoSensitivity; + minIsoChanged(queryControl.minimum); + maxIsoChanged(queryControl.minimum); + } } - } + }); supportedFeaturesChanged(features); } int QGstreamerCamera::setV4L2ColorTemperature(int temperature) { - struct v4l2_control control; - ::memset(&control, 0, sizeof(control)); - if (v4l2AutoWhiteBalanceSupported) { setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, temperature == 0 ? true : false); } else if (temperature == 0) { @@ -714,22 +695,77 @@ int QGstreamerCamera::setV4L2ColorTemperature(int temperature) bool QGstreamerCamera::setV4L2Parameter(quint32 id, qint32 value) { - struct v4l2_control control{id, value}; - if (::ioctl(v4l2FileDescriptor, VIDIOC_S_CTRL, &control) != 0) { - qWarning() << "Unable to set the V4L2 Parameter" << Qt::hex << id << "to" << value << qt_error_string(errno); - return false; - } - return true; + return withV4L2DeviceFileDescriptor([&](int fd) { + v4l2_control control{ id, value }; + if (::ioctl(fd, VIDIOC_S_CTRL, &control) != 0) { + qWarning() << "Unable to set the V4L2 Parameter" << Qt::hex << id << "to" << value + << qt_error_string(errno); + return false; + } + return true; + }); } int QGstreamerCamera::getV4L2Parameter(quint32 id) const { - struct v4l2_control control{id, 0}; - if (::ioctl(v4l2FileDescriptor, VIDIOC_G_CTRL, &control) != 0) { - qWarning() << "Unable to get the V4L2 Parameter" << Qt::hex << id << qt_error_string(errno); - return 0; - } - return control.value; + return withV4L2DeviceFileDescriptor([&](int fd) { + v4l2_control control{ id, 0 }; + if (::ioctl(fd, VIDIOC_G_CTRL, &control) != 0) { + qWarning() << "Unable to get the V4L2 Parameter" << Qt::hex << id + << qt_error_string(errno); + return 0; + } + return control.value; + }); +} + +QGstreamerCustomCamera::QGstreamerCustomCamera(QCamera *camera) + : QGstreamerCameraBase{ + camera, + }, + m_userProvidedGstElement{ + false, + } +{ +} + +QGstreamerCustomCamera::QGstreamerCustomCamera(QCamera *camera, QGstElement element) + : QGstreamerCameraBase{ + camera, + }, + gstCamera{ + std::move(element), + }, + m_userProvidedGstElement{ + true, + } +{ +} + +void QGstreamerCustomCamera::setCamera(const QCameraDevice &device) +{ + if (m_userProvidedGstElement) + return; + + gstCamera = QGstBin::createFromPipelineDescription(device.id(), /*name=*/nullptr, + /* ghostUnlinkedPads=*/true); +} + +bool QGstreamerCustomCamera::isActive() const +{ + return m_active; +} + +void QGstreamerCustomCamera::setActive(bool active) +{ + if (m_active == active) + return; + + m_active = active; + + emit activeChanged(active); } #endif + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h index ac57eb5ef..f43c01f34 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h @@ -1,42 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERCAMERACONTROL_H #define QGSTREAMERCAMERACONTROL_H @@ -52,18 +15,28 @@ // We mean it. // -#include <QHash> #include <private/qplatformcamera_p.h> -#include "qgstreamermediacapture_p.h" -#include <qgst_p.h> +#include <private/qmultimediautils_p.h> + +#include <mediacapture/qgstreamermediacapture_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> QT_BEGIN_NAMESPACE -class QGstreamerCamera : public QPlatformCamera +class QGstreamerCameraBase : public QPlatformCamera { - Q_OBJECT public: - QGstreamerCamera(QCamera *camera); + using QPlatformCamera::QPlatformCamera; + + virtual QGstElement gstElement() const = 0; +}; + +class QGstreamerCamera : public QGstreamerCameraBase +{ +public: + static QMaybe<QPlatformCamera *> create(QCamera *camera); + virtual ~QGstreamerCamera(); bool isActive() const override; @@ -72,7 +45,7 @@ public: void setCamera(const QCameraDevice &camera) override; bool setCameraFormat(const QCameraFormat &format) override; - QGstElement gstElement() const { return gstCameraBin.element(); } + QGstElement gstElement() const override { return gstCameraBin; } #if QT_CONFIG(gstreamer_photography) GstPhotography *photography() const; #endif @@ -96,12 +69,13 @@ public: void setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) override; void setColorTemperature(int temperature) override; - QString v4l2Device() const { return m_v4l2Device; } - bool isV4L2Camera() const { return !m_v4l2Device.isEmpty(); } - private: + QGstreamerCamera(QCamera *camera); + void updateCameraProperties(); + #if QT_CONFIG(linux_v4l) + bool isV4L2Camera() const; void initV4L2Controls(); int setV4L2ColorTemperature(int temperature); bool setV4L2Parameter(quint32 id, qint32 value); @@ -117,7 +91,29 @@ private: qint32 v4l2MaxExposure = 0; qint32 v4l2MinExposureAdjustment = 0; qint32 v4l2MaxExposureAdjustment = 0; - int v4l2FileDescriptor = -1; + + template <typename Functor> + auto withV4L2DeviceFileDescriptor(Functor &&f) const + { + using ReturnType = std::invoke_result_t<Functor, int>; + Q_ASSERT(isV4L2Camera()); + + if (int gstreamerDeviceFd = gstCamera.getInt("device-fd"); gstreamerDeviceFd != -1) + return f(gstreamerDeviceFd); + + auto v4l2FileDescriptor = QFileDescriptorHandle{ + qt_safe_open(m_v4l2DevicePath.toLocal8Bit().constData(), O_RDONLY), + }; + if (!v4l2FileDescriptor) { + qWarning() << "Unable to open the camera" << m_v4l2DevicePath + << "for read to query the parameter info:" << qt_error_string(errno); + if constexpr (std::is_void_v<ReturnType>) + return; + else + return ReturnType{}; + } + return f(v4l2FileDescriptor.get()); + } #endif QCameraDevice m_cameraDevice; @@ -130,7 +126,25 @@ private: QGstElement gstVideoScale; bool m_active = false; - QString m_v4l2Device; + QString m_v4l2DevicePath; +}; + +class QGstreamerCustomCamera : public QGstreamerCameraBase +{ +public: + explicit QGstreamerCustomCamera(QCamera *); + explicit QGstreamerCustomCamera(QCamera *, QGstElement element); + + QGstElement gstElement() const override { return gstCamera; } + void setCamera(const QCameraDevice &) override; + + bool isActive() const override; + void setActive(bool) override; + +private: + QGstElement gstCamera; + bool m_active{}; + const bool m_userProvidedGstElement; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp index 91e48e9c7..9c21dc083 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp @@ -1,68 +1,117 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamerimagecapture_p.h" -#include <private/qplatformcamera_p.h> -#include <private/qplatformimagecapture_p.h> -#include <qgstvideobuffer_p.h> -#include <qgstutils_p.h> -#include <qgstreamermetadata_p.h> -#include <qvideoframeformat.h> -#include <private/qmediastoragelocation_p.h> -#include <QtCore/QDebug> -#include <QtCore/QDir> -#include <qstandardpaths.h> +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/private/qmediastoragelocation_p.h> +#include <QtMultimedia/private/qplatformcamera_p.h> +#include <QtMultimedia/private/qplatformimagecapture_p.h> +#include <QtMultimedia/private/qvideoframe_p.h> +#include <QtCore/qdebug.h> +#include <QtCore/qdir.h> +#include <QtCore/qstandardpaths.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qloggingcategory.h> -#include <qloggingcategory.h> +#include <common/qgstreamermetadata_p.h> +#include <common/qgstvideobuffer_p.h> +#include <common/qgstutils_p.h> + +#include <utility> QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcImageCapture, "qt.multimedia.imageCapture") +namespace { +Q_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture") -QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) - : QPlatformImageCapture(parent), - QGstreamerBufferProbe(ProbeBuffers) +struct ThreadPoolSingleton +{ + QObject m_context; + QMutex m_poolMutex; + QThreadPool *m_instance{}; + bool m_appUnderDestruction = false; + + QThreadPool *get(const QMutexLocker<QMutex> &) + { + if (m_instance) + return m_instance; + if (m_appUnderDestruction || !qApp) + return nullptr; + + using namespace std::chrono; + + m_instance = new QThreadPool(qApp); + m_instance->setMaxThreadCount(1); // 1 thread; + static constexpr auto expiryTimeout = minutes(5); + m_instance->setExpiryTimeout(round<milliseconds>(expiryTimeout).count()); + + QObject::connect(qApp, &QCoreApplication::aboutToQuit, &m_context, [&] { + // we need to make sure that thread-local QRhi is destroyed before the application to + // prevent QTBUG-124189 + QMutexLocker guard(&m_poolMutex); + delete m_instance; + m_instance = {}; + m_appUnderDestruction = true; + }); + + QObject::connect(qApp, &QCoreApplication::destroyed, &m_context, [&] { + m_appUnderDestruction = false; + }); + return m_instance; + } + + template <typename Functor> + QFuture<void> run(Functor &&f) + { + QMutexLocker guard(&m_poolMutex); + QThreadPool *pool = get(guard); + if (!pool) + return QFuture<void>{}; + + return QtConcurrent::run(pool, std::forward<Functor>(f)); + } +}; + +ThreadPoolSingleton s_threadPoolSingleton; + +}; // namespace + +QMaybe<QPlatformImageCapture *> QGstreamerImageCapture::create(QImageCapture *parent) { - bin = QGstBin("imageCaptureBin"); + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "queue", "capsfilter", "videoconvert", "jpegenc", "jifmux", "fakesink"); + if (error) + return *error; + + return new QGstreamerImageCapture(parent); +} - queue = QGstElement("queue", "imageCaptureQueue"); +QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) + : QPlatformImageCapture(parent), + QGstreamerBufferProbe(ProbeBuffers), + bin{ + QGstBin::create("imageCaptureBin"), + }, + queue{ + QGstElement::createFromFactory("queue", "imageCaptureQueue"), + }, + filter{ + QGstElement::createFromFactory("capsfilter", "filter"), + }, + videoConvert{ + QGstElement::createFromFactory("videoconvert", "imageCaptureConvert"), + }, + encoder{ + QGstElement::createFromFactory("jpegenc", "jpegEncoder"), + }, + muxer{ + QGstElement::createFromFactory("jifmux", "jpegMuxer"), + }, + sink{ + QGstElement::createFromFactory("fakesink", "imageCaptureSink"), + } +{ // configures the queue to be fast, lightweight and non blocking queue.set("leaky", 2 /*downstream*/); queue.set("silent", true); @@ -70,37 +119,45 @@ QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) queue.set("max-size-bytes", uint(0)); queue.set("max-size-time", quint64(0)); - videoConvert = QGstElement("videoconvert", "imageCaptureConvert"); - encoder = QGstElement("jpegenc", "jpegEncoder"); - muxer = QGstElement("jifmux", "jpegMuxer"); - sink = QGstElement("fakesink","imageCaptureSink"); // imageCaptureSink do not wait for a preroll buffer when going READY -> PAUSED // as no buffer will arrive until capture() is called sink.set("async", false); - bin.add(queue, videoConvert, encoder, muxer, sink); - queue.link(videoConvert, encoder, muxer, sink); + bin.add(queue, filter, videoConvert, encoder, muxer, sink); + qLinkGstElements(queue, filter, videoConvert, encoder, muxer, sink); bin.addGhostPad(queue, "sink"); addProbeToPad(queue.staticPad("src").pad(), false); sink.set("signal-handoffs", true); - g_signal_connect(sink.object(), "handoff", G_CALLBACK(&QGstreamerImageCapture::saveImageFilter), this); + m_handoffConnection = sink.connect("handoff", G_CALLBACK(&saveImageFilter), this); } QGstreamerImageCapture::~QGstreamerImageCapture() { bin.setStateSync(GST_STATE_NULL); + + // wait for pending futures + auto pendingFutures = [&] { + QMutexLocker guard(&m_mutex); + return std::move(m_pendingFutures); + }(); + + for (QFuture<void> &pendingImage : pendingFutures) + pendingImage.waitForFinished(); } bool QGstreamerImageCapture::isReadyForCapture() const { + QMutexLocker guard(&m_mutex); return m_session && !passImage && cameraActive; } int QGstreamerImageCapture::capture(const QString &fileName) { - QString path = QMediaStorageLocation::generateFileName(fileName, QStandardPaths::PicturesLocation, QLatin1String("jpg")); + using namespace Qt::Literals; + QString path = QMediaStorageLocation::generateFileName( + fileName, QStandardPaths::PicturesLocation, u"jpg"_s); return doCapture(path); } @@ -111,100 +168,153 @@ int QGstreamerImageCapture::captureToBuffer() int QGstreamerImageCapture::doCapture(const QString &fileName) { - qCDebug(qLcImageCapture) << "do capture"; - if (!m_session) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::ResourceError), - Q_ARG(QString, QPlatformImageCapture::msgImageCaptureNotSet())); - - qCDebug(qLcImageCapture) << "error 1"; - return -1; - } - if (!m_session->camera()) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::ResourceError), - Q_ARG(QString,tr("No camera available."))); - - qCDebug(qLcImageCapture) << "error 2"; - return -1; - } - if (passImage) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::NotReadyError), - Q_ARG(QString, QPlatformImageCapture::msgCameraNotReady())); - - qCDebug(qLcImageCapture) << "error 3"; - return -1; - } - m_lastId++; + qCDebug(qLcImageCaptureGst) << "do capture"; + + { + QMutexLocker guard(&m_mutex); + if (!m_session) { + invokeDeferred([this] { + emit error(-1, QImageCapture::ResourceError, + QPlatformImageCapture::msgImageCaptureNotSet()); + }); + + qCDebug(qLcImageCaptureGst) << "error 1"; + return -1; + } + if (!m_session->camera()) { + invokeDeferred([this] { + emit error(-1, QImageCapture::ResourceError, tr("No camera available.")); + }); - pendingImages.enqueue({m_lastId, fileName, QMediaMetaData{}}); - // let one image pass the pipeline - passImage = true; + qCDebug(qLcImageCaptureGst) << "error 2"; + return -1; + } + if (passImage) { + invokeDeferred([this] { + emit error(-1, QImageCapture::NotReadyError, + QPlatformImageCapture::msgCameraNotReady()); + }); + + qCDebug(qLcImageCaptureGst) << "error 3"; + return -1; + } + m_lastId++; + + pendingImages.enqueue({ m_lastId, fileName, QMediaMetaData{} }); + // let one image pass the pipeline + passImage = true; + } emit readyForCaptureChanged(false); return m_lastId; } +void QGstreamerImageCapture::setResolution(const QSize &resolution) +{ + QGstCaps padCaps = bin.staticPad("sink").currentCaps(); + if (padCaps.isNull()) { + qDebug() << "Camera not ready"; + return; + } + QGstCaps caps = padCaps.copy(); + if (caps.isNull()) + return; + + gst_caps_set_simple(caps.caps(), "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT, + resolution.height(), nullptr); + filter.set("caps", caps); +} + +// HACK: gcc-10 and earlier reject [=,this] when building with c++17 +#if __cplusplus >= 202002L +# define EQ_THIS_CAPTURE =, this +#else +# define EQ_THIS_CAPTURE = +#endif + bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer) { + QMutexLocker guard(&m_mutex); + if (!passImage) return false; - qCDebug(qLcImageCapture) << "probe buffer"; + qCDebug(qLcImageCaptureGst) << "probe buffer"; - passImage = false; + QGstBufferHandle bufferHandle{ + buffer, + QGstBufferHandle::NeedsRef, + }; - emit readyForCaptureChanged(isReadyForCapture()); + passImage = false; - QGstCaps caps = gst_pad_get_current_caps(bin.staticPad("sink").pad()); - GstVideoInfo previewInfo; - gst_video_info_from_caps(&previewInfo, caps.get()); + bool ready = isReadyForCapture(); + invokeDeferred([this, ready] { + emit readyForCaptureChanged(ready); + }); + QGstCaps caps = bin.staticPad("sink").currentCaps(); auto memoryFormat = caps.memoryFormat(); - auto fmt = QGstCaps(caps).formatForCaps(&previewInfo); - auto *sink = m_session->gstreamerVideoSink(); - auto *gstBuffer = new QGstVideoBuffer(buffer, previewInfo, sink, fmt, memoryFormat); - QVideoFrame frame(gstBuffer, fmt); - QImage img = frame.toImage(); - if (img.isNull()) { - qDebug() << "received a null image"; - return true; - } - auto &imageData = pendingImages.head(); - - emit imageExposed(imageData.id); - - qCDebug(qLcImageCapture) << "Image available!"; - emit imageAvailable(imageData.id, frame); + GstVideoInfo previewInfo; + QVideoFrameFormat fmt; + auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo(); + if (optionalFormatAndVideoInfo) + std::tie(fmt, previewInfo) = std::move(*optionalFormatAndVideoInfo); + + int futureId = futureIDAllocator += 1; + + // ensure QVideoFrame::toImage is executed on a worker thread that is joined before the + // qApplication is destroyed + QFuture<void> future = s_threadPoolSingleton.run([EQ_THIS_CAPTURE]() mutable { + QMutexLocker guard(&m_mutex); + auto scopeExit = qScopeGuard([&] { + m_pendingFutures.remove(futureId); + }); + + if (!m_session) { + qDebug() << "QGstreamerImageCapture::probeBuffer: no session"; + return; + } - emit imageCaptured(imageData.id, img); + auto *sink = m_session->gstreamerVideoSink(); + auto gstBuffer = std::make_unique<QGstVideoBuffer>(std::move(bufferHandle), previewInfo, + sink, fmt, memoryFormat); - QMediaMetaData metaData = this->metaData(); - metaData.insert(QMediaMetaData::Date, QDateTime::currentDateTime()); - metaData.insert(QMediaMetaData::Resolution, frame.size()); - imageData.metaData = metaData; + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(gstBuffer), fmt); + QImage img = frame.toImage(); + if (img.isNull()) { + qDebug() << "received a null image"; + return; + } - // ensure taginject injects this metaData - const auto &md = static_cast<const QGstreamerMetaData &>(metaData); - md.setMetaData(muxer.element()); + QMediaMetaData imageMetaData = metaData(); + imageMetaData.insert(QMediaMetaData::Resolution, frame.size()); + pendingImages.head().metaData = std::move(imageMetaData); + PendingImage pendingImage = pendingImages.head(); + + invokeDeferred([this, pendingImage = std::move(pendingImage), frame = std::move(frame), + img = std::move(img)]() mutable { + emit imageExposed(pendingImage.id); + qCDebug(qLcImageCaptureGst) << "Image available!"; + emit imageAvailable(pendingImage.id, frame); + emit imageCaptured(pendingImage.id, img); + emit imageMetadataAvailable(pendingImage.id, pendingImage.metaData); + }); + }); + + if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable + return true; - emit imageMetadataAvailable(imageData.id, metaData); + m_pendingFutures.insert(futureId, future); return true; } +#undef EQ_THIS_CAPTURE + void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session) { + QMutexLocker guard(&m_mutex); QGstreamerMediaCapture *captureSession = static_cast<QGstreamerMediaCapture *>(session); if (m_session == captureSession) return; @@ -225,71 +335,98 @@ void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *ses return; } - connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, &QGstreamerImageCapture::onCameraChanged); + connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, + &QGstreamerImageCapture::onCameraChanged); onCameraChanged(); } +void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m) +{ + { + QMutexLocker guard(&m_mutex); + QPlatformImageCapture::setMetaData(m); + } + + // ensure taginject injects this metaData + applyMetaDataToTagSetter(m, muxer); +} + void QGstreamerImageCapture::cameraActiveChanged(bool active) { - qCDebug(qLcImageCapture) << "cameraActiveChanged" << cameraActive << active; + qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active; if (cameraActive == active) return; cameraActive = active; - qCDebug(qLcImageCapture) << "isReady" << isReadyForCapture(); + qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture(); emit readyForCaptureChanged(isReadyForCapture()); } void QGstreamerImageCapture::onCameraChanged() { + QMutexLocker guard(&m_mutex); if (m_session->camera()) { cameraActiveChanged(m_session->camera()->isActive()); - connect(m_session->camera(), &QPlatformCamera::activeChanged, this, &QGstreamerImageCapture::cameraActiveChanged); + connect(m_session->camera(), &QPlatformCamera::activeChanged, this, + &QGstreamerImageCapture::cameraActiveChanged); } else { cameraActiveChanged(false); } } -gboolean QGstreamerImageCapture::saveImageFilter(GstElement *element, - GstBuffer *buffer, - GstPad *pad, - void *appdata) +gboolean QGstreamerImageCapture::saveImageFilter(GstElement *, GstBuffer *buffer, GstPad *, + QGstreamerImageCapture *capture) { - Q_UNUSED(element); - Q_UNUSED(pad); - QGstreamerImageCapture *capture = static_cast<QGstreamerImageCapture *>(appdata); + capture->saveBufferToImage(buffer); + return true; +} - capture->passImage = false; +void QGstreamerImageCapture::saveBufferToImage(GstBuffer *buffer) +{ + QMutexLocker guard(&m_mutex); + passImage = false; - if (capture->pendingImages.isEmpty()) { - return true; - } + if (pendingImages.isEmpty()) + return; - auto imageData = capture->pendingImages.dequeue(); - if (imageData.filename.isEmpty()) { - return true; - } + PendingImage imageData = pendingImages.dequeue(); + if (imageData.filename.isEmpty()) + return; - qCDebug(qLcImageCapture) << "saving image as" << imageData.filename; + int id = futureIDAllocator++; + QGstBufferHandle bufferHandle{ + buffer, + QGstBufferHandle::NeedsRef, + }; + + QFuture<void> saveImageFuture = QtConcurrent::run([this, imageData, bufferHandle, + id]() mutable { + auto cleanup = qScopeGuard([&] { + QMutexLocker guard(&m_mutex); + m_pendingFutures.remove(id); + }); + + qCDebug(qLcImageCaptureGst) << "saving image as" << imageData.filename; + + QFile f(imageData.filename); + if (!f.open(QFile::WriteOnly)) { + qCDebug(qLcImageCaptureGst) << " could not open image file for writing"; + return; + } - QFile f(imageData.filename); - if (f.open(QFile::WriteOnly)) { GstMapInfo info; + GstBuffer *buffer = bufferHandle.get(); if (gst_buffer_map(buffer, &info, GST_MAP_READ)) { f.write(reinterpret_cast<const char *>(info.data), info.size); gst_buffer_unmap(buffer, &info); } f.close(); - static QMetaMethod savedSignal = QMetaMethod::fromSignal(&QGstreamerImageCapture::imageSaved); - savedSignal.invoke(capture, - Qt::QueuedConnection, - Q_ARG(int, imageData.id), - Q_ARG(QString, imageData.filename)); - } else { - qCDebug(qLcImageCapture) << " could not open image file for writing"; - } + QMetaObject::invokeMethod(this, [this, imageData = std::move(imageData)]() mutable { + emit imageSaved(imageData.id, imageData.filename); + }); + }); - return TRUE; + m_pendingFutures.insert(id, saveImageFuture); } QImageEncoderSettings QGstreamerImageCapture::imageSettings() const @@ -300,9 +437,14 @@ QImageEncoderSettings QGstreamerImageCapture::imageSettings() const void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings) { if (m_settings != settings) { + QSize resolution = settings.resolution(); + if (m_settings.resolution() != resolution && !resolution.isEmpty()) + setResolution(resolution); + m_settings = settings; - // ### } } QT_END_NAMESPACE + +#include "moc_qgstreamerimagecapture_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h index 1f498fe0b..04a7c00b4 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h @@ -1,42 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERIMAGECAPTURECONTROL_H #define QGSTREAMERIMAGECAPTURECONTROL_H @@ -52,23 +15,26 @@ // We mean it. // -#include <private/qplatformimagecapture_p.h> -#include "qgstreamermediacapture_p.h" -#include "qgstreamerbufferprobe_p.h" +#include <QtMultimedia/private/qplatformimagecapture_p.h> +#include <QtMultimedia/private/qmultimediautils_p.h> -#include <qqueue.h> +#include <QtCore/qmutex.h> +#include <QtCore/qqueue.h> +#include <QtConcurrent/QtConcurrentRun> -#include <qgst_p.h> +#include <common/qgst_p.h> +#include <common/qgstreamerbufferprobe_p.h> +#include <mediacapture/qgstreamermediacapture_p.h> #include <gst/video/video.h> QT_BEGIN_NAMESPACE class QGstreamerImageCapture : public QPlatformImageCapture, private QGstreamerBufferProbe - { Q_OBJECT + public: - QGstreamerImageCapture(QImageCapture *parent); + static QMaybe<QPlatformImageCapture *> create(QImageCapture *parent); virtual ~QGstreamerImageCapture(); bool isReadyForCapture() const override; @@ -82,16 +48,26 @@ public: void setCaptureSession(QPlatformMediaCaptureSession *session); - QGstElement gstElement() const { return bin.element(); } + QGstElement gstElement() const { return bin; } + + void setMetaData(const QMediaMetaData &m) override; public Q_SLOTS: void cameraActiveChanged(bool active); void onCameraChanged(); private: + QGstreamerImageCapture(QImageCapture *parent); + + void setResolution(const QSize &resolution); int doCapture(const QString &fileName); - static gboolean saveImageFilter(GstElement *element, GstBuffer *buffer, GstPad *pad, void *appdata); + static gboolean saveImageFilter(GstElement *element, GstBuffer *buffer, GstPad *pad, + QGstreamerImageCapture *capture); + + void saveBufferToImage(GstBuffer *buffer); + mutable QRecursiveMutex + m_mutex; // guard all elements accessed from probeBuffer/saveBufferToImage QGstreamerMediaCapture *m_session = nullptr; int m_lastId = 0; QImageEncoderSettings m_settings; @@ -106,6 +82,7 @@ private: QGstBin bin; QGstElement queue; + QGstElement filter; QGstElement videoConvert; QGstElement encoder; QGstElement muxer; @@ -114,6 +91,17 @@ private: bool passImage = false; bool cameraActive = false; + + QGObjectHandlerScopedConnection m_handoffConnection; + + QMap<int, QFuture<void>> m_pendingFutures; + int futureIDAllocator = 0; + + template <typename Functor> + void invokeDeferred(Functor &&fn) + { + QMetaObject::invokeMethod(this, std::forward<decltype(fn)>(fn), Qt::QueuedConnection); + }; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp index 15cd076e9..d807ab804 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp @@ -1,58 +1,19 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamermediacapture_p.h" -#include "qgstreamermediaencoder_p.h" -#include "qgstreamerimagecapture_p.h" -#include "qgstreamercamera_p.h" -#include <qgstpipeline_p.h> - -#include "qgstreameraudioinput_p.h" -#include "qgstreameraudiooutput_p.h" -#include "qgstreamervideooutput_p.h" - -#include <qloggingcategory.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -QT_BEGIN_NAMESPACE +#include <mediacapture/qgstreamermediacapture_p.h> +#include <mediacapture/qgstreamermediaencoder_p.h> +#include <mediacapture/qgstreamerimagecapture_p.h> +#include <mediacapture/qgstreamercamera_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreameraudioinput_p.h> +#include <common/qgstreameraudiooutput_p.h> +#include <common/qgstreamervideooutput_p.h> -Q_LOGGING_CATEGORY(qLcMediaCapture, "qt.multimedia.capture") +#include <QtCore/qloggingcategory.h> +#include <QtCore/private/quniquehandle_p.h> +QT_BEGIN_NAMESPACE static void linkTeeToPad(QGstElement tee, QGstPad sink) { @@ -63,37 +24,42 @@ static void linkTeeToPad(QGstElement tee, QGstPad sink) source.link(sink); } -static void unlinkTeeFromPad(QGstElement tee, QGstPad sink) +QMaybe<QPlatformMediaCaptureSession *> QGstreamerMediaCapture::create() { - if (tee.isNull() || sink.isNull()) - return; + auto videoOutput = QGstreamerVideoOutput::create(); + if (!videoOutput) + return videoOutput.error(); - auto source = sink.peer(); - source.unlink(sink); + static const auto error = qGstErrorMessageIfElementsNotAvailable("tee", "capsfilter"); + if (error) + return *error; - tee.releaseRequestPad(source); + return new QGstreamerMediaCapture(videoOutput.value()); } - -QGstreamerMediaCapture::QGstreamerMediaCapture() - : gstPipeline("pipeline") +QGstreamerMediaCapture::QGstreamerMediaCapture(QGstreamerVideoOutput *videoOutput) + : capturePipeline(QGstPipeline::create("mediaCapturePipeline")), gstVideoOutput(videoOutput) { - gstVideoOutput = new QGstreamerVideoOutput(this); + gstVideoOutput->setParent(this); gstVideoOutput->setIsPreview(); - gstVideoOutput->setPipeline(gstPipeline); + gstVideoOutput->setPipeline(capturePipeline); // Use system clock to drive all elements in the pipeline. Otherwise, // the clock is sourced from the elements (e.g. from an audio source). // Since the elements are added and removed dynamically the clock would // also change causing lost of synchronization in the pipeline. - gst_pipeline_use_clock(gstPipeline.pipeline(), gst_system_clock_obtain()); + + QGstClockHandle systemClock{ + gst_system_clock_obtain(), + }; + gst_pipeline_use_clock(capturePipeline.pipeline(), systemClock.get()); // This is the recording pipeline with only live sources, thus the pipeline // will be always in the playing state. - gstPipeline.setState(GST_STATE_PLAYING); - gstPipeline.setInStoppedState(false); + capturePipeline.setState(GST_STATE_PLAYING); + gstVideoOutput->setActive(true); - gstPipeline.dumpGraph("initial"); + capturePipeline.dumpGraph("initial"); } QGstreamerMediaCapture::~QGstreamerMediaCapture() @@ -101,7 +67,7 @@ QGstreamerMediaCapture::~QGstreamerMediaCapture() setMediaRecorder(nullptr); setImageCapture(nullptr); setCamera(nullptr); - gstPipeline.setStateSync(GST_STATE_NULL); + capturePipeline.setStateSync(GST_STATE_NULL); } QPlatformCamera *QGstreamerMediaCapture::camera() @@ -109,52 +75,64 @@ QPlatformCamera *QGstreamerMediaCapture::camera() return gstCamera; } -void QGstreamerMediaCapture::setCamera(QPlatformCamera *camera) +void QGstreamerMediaCapture::setCamera(QPlatformCamera *platformCamera) { - QGstreamerCamera *control = static_cast<QGstreamerCamera *>(camera); - if (gstCamera == control) + auto *camera = static_cast<QGstreamerCameraBase *>(platformCamera); + if (gstCamera == camera) return; if (gstCamera) { - unlinkTeeFromPad(gstVideoTee, encoderVideoSink); - unlinkTeeFromPad(gstVideoTee, imageCaptureSink); + QObject::disconnect(gstCameraActiveConnection); + if (gstVideoTee) + setCameraActive(false); + } - auto camera = gstCamera->gstElement(); + gstCamera = camera; - gstPipeline.remove(camera); - gstPipeline.remove(gstVideoTee); - gstPipeline.remove(gstVideoOutput->gstElement()); + if (gstCamera) { + gstCameraActiveConnection = QObject::connect(camera, &QPlatformCamera::activeChanged, this, + &QGstreamerMediaCapture::setCameraActive); + if (gstCamera->isActive()) + setCameraActive(true); + } - camera.setStateSync(GST_STATE_NULL); - gstVideoTee.setStateSync(GST_STATE_NULL); - gstVideoOutput->gstElement().setStateSync(GST_STATE_NULL); + emit cameraChanged(); +} - gstVideoTee = {}; - gstCamera->setCaptureSession(nullptr); - } +void QGstreamerMediaCapture::setCameraActive(bool activate) +{ + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (activate) { + QGstElement cameraElement = gstCamera->gstElement(); + gstVideoTee = QGstElement::createFromFactory("tee", "videotee"); + gstVideoTee.set("allow-not-linked", true); - gstCamera = control; - if (gstCamera) { - QGstElement camera = gstCamera->gstElement(); - gstVideoTee = QGstElement("tee", "videotee"); - gstVideoTee.set("allow-not-linked", true); + capturePipeline.add(gstVideoOutput->gstElement(), cameraElement, gstVideoTee); - gstPipeline.add(gstVideoOutput->gstElement(), camera, gstVideoTee); + linkTeeToPad(gstVideoTee, encoderVideoSink); + linkTeeToPad(gstVideoTee, gstVideoOutput->gstElement().staticPad("sink")); + linkTeeToPad(gstVideoTee, imageCaptureSink); - linkTeeToPad(gstVideoTee, encoderVideoSink); - linkTeeToPad(gstVideoTee, gstVideoOutput->gstElement().staticPad("sink")); - linkTeeToPad(gstVideoTee, imageCaptureSink); + qLinkGstElements(cameraElement, gstVideoTee); - camera.link(gstVideoTee); + capturePipeline.syncChildrenState(); + } else { + if (encoderVideoCapsFilter) + qUnlinkGstElements(gstVideoTee, encoderVideoCapsFilter); + if (m_imageCapture) + qUnlinkGstElements(gstVideoTee, m_imageCapture->gstElement()); - gstVideoOutput->gstElement().setState(GST_STATE_PLAYING); - gstVideoTee.setState(GST_STATE_PLAYING); - camera.setState(GST_STATE_PLAYING); - } + auto camera = gstCamera->gstElement(); - gstPipeline.dumpGraph("camera"); + capturePipeline.stopAndRemoveElements(camera, gstVideoTee, + gstVideoOutput->gstElement()); - emit cameraChanged(); + gstVideoTee = {}; + gstCamera->setCaptureSession(nullptr); + } + }); + + capturePipeline.dumpGraph("camera"); } QPlatformImageCapture *QGstreamerMediaCapture::imageCapture() @@ -168,24 +146,25 @@ void QGstreamerMediaCapture::setImageCapture(QPlatformImageCapture *imageCapture if (m_imageCapture == control) return; - if (m_imageCapture) { - unlinkTeeFromPad(gstVideoTee, imageCaptureSink); - gstPipeline.remove(m_imageCapture->gstElement()); - m_imageCapture->gstElement().setStateSync(GST_STATE_NULL); - imageCaptureSink = {}; - m_imageCapture->setCaptureSession(nullptr); - } + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (m_imageCapture) { + qUnlinkGstElements(gstVideoTee, m_imageCapture->gstElement()); + capturePipeline.stopAndRemoveElements(m_imageCapture->gstElement()); + imageCaptureSink = {}; + m_imageCapture->setCaptureSession(nullptr); + } - m_imageCapture = control; - if (m_imageCapture) { - imageCaptureSink = m_imageCapture->gstElement().staticPad("sink"); - m_imageCapture->gstElement().setState(GST_STATE_PLAYING); - gstPipeline.add(m_imageCapture->gstElement()); - linkTeeToPad(gstVideoTee, imageCaptureSink); - m_imageCapture->setCaptureSession(this); - } + m_imageCapture = control; + if (m_imageCapture) { + imageCaptureSink = m_imageCapture->gstElement().staticPad("sink"); + capturePipeline.add(m_imageCapture->gstElement()); + m_imageCapture->gstElement().syncStateWithParent(); + linkTeeToPad(gstVideoTee, imageCaptureSink); + m_imageCapture->setCaptureSession(this); + } + }); - gstPipeline.dumpGraph("imageCapture"); + capturePipeline.dumpGraph("imageCapture"); emit imageCaptureChanged(); } @@ -203,7 +182,7 @@ void QGstreamerMediaCapture::setMediaRecorder(QPlatformMediaRecorder *recorder) m_mediaEncoder->setCaptureSession(this); emit encoderChanged(); - gstPipeline.dumpGraph("encoder"); + capturePipeline.dumpGraph("encoder"); } QPlatformMediaRecorder *QGstreamerMediaCapture::mediaRecorder() @@ -213,55 +192,62 @@ QPlatformMediaRecorder *QGstreamerMediaCapture::mediaRecorder() void QGstreamerMediaCapture::linkEncoder(QGstPad audioSink, QGstPad videoSink) { - if (!gstVideoTee.isNull() && !videoSink.isNull()) { - auto caps = gst_pad_get_current_caps(gstVideoTee.sink().pad()); + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (!gstVideoTee.isNull() && !videoSink.isNull()) { + QGstCaps caps = gstVideoTee.sink().currentCaps(); - encoderVideoCapsFilter = QGstElement("capsfilter", "encoderVideoCapsFilter"); - encoderVideoCapsFilter.set("caps", QGstMutableCaps(caps)); + encoderVideoCapsFilter = + QGstElement::createFromFactory("capsfilter", "encoderVideoCapsFilter"); + Q_ASSERT(encoderVideoCapsFilter); + encoderVideoCapsFilter.set("caps", caps); - gstPipeline.add(encoderVideoCapsFilter); + capturePipeline.add(encoderVideoCapsFilter); - encoderVideoCapsFilter.src().link(videoSink); - linkTeeToPad(gstVideoTee, encoderVideoCapsFilter.sink()); - encoderVideoCapsFilter.setState(GST_STATE_PLAYING); - encoderVideoSink = encoderVideoCapsFilter.sink(); - } + encoderVideoCapsFilter.src().link(videoSink); + linkTeeToPad(gstVideoTee, encoderVideoCapsFilter.sink()); + encoderVideoSink = encoderVideoCapsFilter.sink(); + } - if (!gstAudioTee.isNull() && !audioSink.isNull()) { - auto caps = gst_pad_get_current_caps(gstAudioTee.sink().pad()); + if (!gstAudioTee.isNull() && !audioSink.isNull()) { + QGstCaps caps = gstAudioTee.sink().currentCaps(); - encoderAudioCapsFilter = QGstElement("capsfilter", "encoderAudioCapsFilter"); - encoderAudioCapsFilter.set("caps", QGstMutableCaps(caps)); + encoderAudioCapsFilter = + QGstElement::createFromFactory("capsfilter", "encoderAudioCapsFilter"); + Q_ASSERT(encoderAudioCapsFilter); + encoderAudioCapsFilter.set("caps", caps); - gstPipeline.add(encoderAudioCapsFilter); + capturePipeline.add(encoderAudioCapsFilter); - encoderAudioCapsFilter.src().link(audioSink); - linkTeeToPad(gstAudioTee, encoderAudioCapsFilter.sink()); - encoderAudioCapsFilter.setState(GST_STATE_PLAYING); - encoderAudioSink = encoderAudioCapsFilter.sink(); - } + encoderAudioCapsFilter.src().link(audioSink); + linkTeeToPad(gstAudioTee, encoderAudioCapsFilter.sink()); + encoderAudioSink = encoderAudioCapsFilter.sink(); + } + }); } void QGstreamerMediaCapture::unlinkEncoder() { - if (!encoderVideoCapsFilter.isNull()) { - encoderVideoCapsFilter.src().unlinkPeer(); - unlinkTeeFromPad(gstVideoTee, encoderVideoCapsFilter.sink()); - gstPipeline.remove(encoderVideoCapsFilter); - encoderVideoCapsFilter.setStateSync(GST_STATE_NULL); - encoderVideoCapsFilter = {}; - } + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (encoderVideoCapsFilter) { + qUnlinkGstElements(gstVideoTee, encoderVideoCapsFilter); + capturePipeline.stopAndRemoveElements(encoderVideoCapsFilter); + encoderVideoCapsFilter = {}; + } - if (!encoderAudioCapsFilter.isNull()) { - encoderAudioCapsFilter.src().unlinkPeer(); - unlinkTeeFromPad(gstAudioTee, encoderAudioCapsFilter.sink()); - gstPipeline.remove(encoderAudioCapsFilter); - encoderAudioCapsFilter.setStateSync(GST_STATE_NULL); - encoderAudioCapsFilter = {}; - } + if (encoderAudioCapsFilter) { + qUnlinkGstElements(gstAudioTee, encoderAudioCapsFilter); + capturePipeline.stopAndRemoveElements(encoderAudioCapsFilter); + encoderAudioCapsFilter = {}; + } + + encoderAudioSink = {}; + encoderVideoSink = {}; + }); +} - encoderAudioSink = {}; - encoderVideoSink = {}; +const QGstPipeline &QGstreamerMediaCapture::pipeline() const +{ + return capturePipeline; } void QGstreamerMediaCapture::setAudioInput(QPlatformAudioInput *input) @@ -269,41 +255,39 @@ void QGstreamerMediaCapture::setAudioInput(QPlatformAudioInput *input) if (gstAudioInput == input) return; - if (gstAudioInput) { - unlinkTeeFromPad(gstAudioTee, encoderAudioSink); + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (gstAudioInput) { + if (encoderAudioCapsFilter) + qUnlinkGstElements(gstAudioTee, encoderAudioCapsFilter); + + if (gstAudioOutput) { + qUnlinkGstElements(gstAudioTee, gstAudioOutput->gstElement()); + capturePipeline.stopAndRemoveElements(gstAudioOutput->gstElement()); + } - if (gstAudioOutput) { - unlinkTeeFromPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - gstPipeline.remove(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setStateSync(GST_STATE_NULL); + capturePipeline.stopAndRemoveElements(gstAudioInput->gstElement(), gstAudioTee); + gstAudioTee = {}; } - gstPipeline.remove(gstAudioInput->gstElement()); - gstPipeline.remove(gstAudioTee); - gstAudioInput->gstElement().setStateSync(GST_STATE_NULL); - gstAudioTee.setStateSync(GST_STATE_NULL); - gstAudioTee = {}; - } + gstAudioInput = static_cast<QGstreamerAudioInput *>(input); + if (gstAudioInput) { + Q_ASSERT(gstAudioTee.isNull()); + gstAudioTee = QGstElement::createFromFactory("tee", "audiotee"); + gstAudioTee.set("allow-not-linked", true); + capturePipeline.add(gstAudioInput->gstElement(), gstAudioTee); + qLinkGstElements(gstAudioInput->gstElement(), gstAudioTee); - gstAudioInput = static_cast<QGstreamerAudioInput *>(input); - if (gstAudioInput) { - Q_ASSERT(gstAudioTee.isNull()); - gstAudioTee = QGstElement("tee", "audiotee"); - gstAudioTee.set("allow-not-linked", true); - gstPipeline.add(gstAudioInput->gstElement(), gstAudioTee); - gstAudioInput->gstElement().link(gstAudioTee); - - if (gstAudioOutput) { - gstPipeline.add(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setState(GST_STATE_PLAYING); - linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - } + if (gstAudioOutput) { + capturePipeline.add(gstAudioOutput->gstElement()); + gstAudioOutput->gstElement().setState(GST_STATE_PLAYING); + linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); + } - gstAudioTee.setState(GST_STATE_PLAYING); - gstAudioInput->gstElement().setStateSync(GST_STATE_PLAYING); + capturePipeline.syncChildrenState(); - linkTeeToPad(gstAudioTee, encoderAudioSink); - } + linkTeeToPad(gstAudioTee, encoderAudioSink); + } + }); } void QGstreamerMediaCapture::setVideoPreview(QVideoSink *sink) @@ -316,19 +300,20 @@ void QGstreamerMediaCapture::setAudioOutput(QPlatformAudioOutput *output) if (gstAudioOutput == output) return; - if (gstAudioOutput && gstAudioInput) { - // If audio input is set, the output is in the pipeline - unlinkTeeFromPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - gstPipeline.remove(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setStateSync(GST_STATE_NULL); - } + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (gstAudioOutput && gstAudioInput) { + // If audio input is set, the output is in the pipeline + qUnlinkGstElements(gstAudioTee, gstAudioOutput->gstElement()); + capturePipeline.stopAndRemoveElements(gstAudioOutput->gstElement()); + } - gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); - if (gstAudioOutput && gstAudioInput) { - gstPipeline.add(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setState(GST_STATE_PLAYING); - linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - } + gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); + if (gstAudioOutput && gstAudioInput) { + capturePipeline.add(gstAudioOutput->gstElement()); + capturePipeline.syncChildrenState(); + linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); + } + }); } QGstreamerVideoSink *QGstreamerMediaCapture::gstreamerVideoSink() const @@ -336,5 +321,6 @@ QGstreamerVideoSink *QGstreamerMediaCapture::gstreamerVideoSink() const return gstVideoOutput ? gstVideoOutput->gstreamerVideoSink() : nullptr; } - QT_END_NAMESPACE + +#include "moc_qgstreamermediacapture_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h index 8a7b03f02..c44e31f0e 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERCAPTURESERVICE_H #define QGSTREAMERCAPTURESERVICE_H @@ -54,14 +18,14 @@ #include <private/qplatformmediacapture_p.h> #include <private/qplatformmediaintegration_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> #include <qtimer.h> QT_BEGIN_NAMESPACE -class QGstreamerCamera; +class QGstreamerCameraBase; class QGstreamerImageCapture; class QGstreamerMediaEncoder; class QGstreamerAudioInput; @@ -69,12 +33,12 @@ class QGstreamerAudioOutput; class QGstreamerVideoOutput; class QGstreamerVideoSink; -class QGstreamerMediaCapture : public QPlatformMediaCaptureSession +class QGstreamerMediaCapture final : public QPlatformMediaCaptureSession { Q_OBJECT public: - QGstreamerMediaCapture(); + static QMaybe<QPlatformMediaCaptureSession *> create(); virtual ~QGstreamerMediaCapture(); QPlatformCamera *camera() override; @@ -95,17 +59,22 @@ public: void linkEncoder(QGstPad audioSink, QGstPad videoSink); void unlinkEncoder(); - QGstPipeline pipeline() const { return gstPipeline; } + const QGstPipeline &pipeline() const; QGstreamerVideoSink *gstreamerVideoSink() const; private: + void setCameraActive(bool activate); + + explicit QGstreamerMediaCapture(QGstreamerVideoOutput *videoOutput); + friend QGstreamerMediaEncoder; // Gst elements - QGstPipeline gstPipeline; + QGstPipeline capturePipeline; QGstreamerAudioInput *gstAudioInput = nullptr; - QGstreamerCamera *gstCamera = nullptr; + QGstreamerCameraBase *gstCamera = nullptr; + QMetaObject::Connection gstCameraActiveConnection; QGstElement gstAudioTee; QGstElement gstVideoTee; diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp index 387ba9a71..4ec10ca84 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp @@ -1,63 +1,31 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamermediaencoder_p.h" -#include "qgstreamerintegration_p.h" -#include "qgstreamerformatinfo_p.h" -#include "qgstpipeline_p.h" -#include "qgstreamermessage_p.h" -#include <private/qplatformcamera_p.h> -#include "qaudiodevice.h" -#include <private/qmediastoragelocation_p.h> - -#include <qdebug.h> -#include <qeventloop.h> -#include <qstandardpaths.h> -#include <qmimetype.h> -#include <qloggingcategory.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <mediacapture/qgstreamermediaencoder_p.h> +#include <qgstreamerformatinfo_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgst_debug_p.h> +#include <qgstreamerintegration_p.h> + +#include <QtMultimedia/private/qmediastoragelocation_p.h> +#include <QtMultimedia/private/qplatformcamera_p.h> +#include <QtMultimedia/qaudiodevice.h> + +#include <QtCore/qdebug.h> +#include <QtCore/qeventloop.h> +#include <QtCore/qstandardpaths.h> +#include <QtCore/qmimetype.h> +#include <QtCore/qloggingcategory.h> #include <gst/gsttagsetter.h> #include <gst/gstversion.h> #include <gst/video/video.h> #include <gst/pbutils/encoding-profile.h> -Q_LOGGING_CATEGORY(qLcMediaEncoder, "qt.multimedia.encoder") +static Q_LOGGING_CATEGORY(qLcMediaEncoderGst, "qt.multimedia.encoder") + +QT_BEGIN_NAMESPACE QGstreamerMediaEncoder::QGstreamerMediaEncoder(QMediaRecorder *parent) : QPlatformMediaRecorder(parent), @@ -65,15 +33,17 @@ QGstreamerMediaEncoder::QGstreamerMediaEncoder(QMediaRecorder *parent) videoPauseControl(*this) { signalDurationChangedTimer.setInterval(100); - signalDurationChangedTimer.callOnTimeout([this](){ durationChanged(duration()); }); + signalDurationChangedTimer.callOnTimeout(&signalDurationChangedTimer, [this]() { + durationChanged(duration()); + }); } QGstreamerMediaEncoder::~QGstreamerMediaEncoder() { - if (!gstPipeline.isNull()) { + if (!capturePipeline.isNull()) { finalize(); - gstPipeline.removeMessageFilter(this); - gstPipeline.setStateSync(GST_STATE_NULL); + capturePipeline.removeMessageFilter(this); + capturePipeline.setStateSync(GST_STATE_NULL); } } @@ -84,52 +54,60 @@ bool QGstreamerMediaEncoder::isLocationWritable(const QUrl &) const void QGstreamerMediaEncoder::handleSessionError(QMediaRecorder::Error code, const QString &description) { - error(code, description); + updateError(code, description); stop(); } -bool QGstreamerMediaEncoder::processBusMessage(const QGstreamerMessage &message) +bool QGstreamerMediaEncoder::processBusMessage(const QGstreamerMessage &msg) { - if (message.isNull()) - return false; - auto msg = message; - -// qCDebug(qLcMediaEncoder) << "received event from" << message.source().name() << Qt::hex << message.type(); -// if (message.type() == GST_MESSAGE_STATE_CHANGED) { -// GstState oldState; -// GstState newState; -// GstState pending; -// gst_message_parse_state_changed(gm, &oldState, &newState, &pending); -// qCDebug(qLcMediaEncoder) << "received state change from" << message.source().name() << oldState << newState << pending; -// } - if (msg.type() == GST_MESSAGE_ELEMENT) { - QGstStructure s = msg.structure(); - qCDebug(qLcMediaEncoder) << "received element message from" << msg.source().name() << s.name(); + constexpr bool traceStateChange = false; + constexpr bool traceAllEvents = false; + + if constexpr (traceAllEvents) + qCDebug(qLcMediaEncoderGst) << "received event:" << msg; + + switch (msg.type()) { + case GST_MESSAGE_ELEMENT: { + QGstStructureView s = msg.structure(); if (s.name() == "GstBinForwarded") - msg = QGstreamerMessage(s); - if (msg.isNull()) - return false; + return processBusMessage(s.getMessage()); + + qCDebug(qLcMediaEncoderGst) + << "received element message from" << msg.source().name() << s.name(); + return false; } - if (msg.type() == GST_MESSAGE_EOS) { - qCDebug(qLcMediaEncoder) << "received EOS from" << msg.source().name(); + case GST_MESSAGE_EOS: { + qCDebug(qLcMediaEncoderGst) << "received EOS from" << msg.source().name(); finalize(); return false; } - if (msg.type() == GST_MESSAGE_ERROR) { - GError *err; - gchar *debug; - gst_message_parse_error(msg.rawMessage(), &err, &debug); - error(QMediaRecorder::ResourceError, QString::fromUtf8(err->message)); - g_error_free(err); - g_free(debug); + case GST_MESSAGE_ERROR: { + qCDebug(qLcMediaEncoderGst) + << "received error:" << msg.source().name() << QCompactGstMessageAdaptor(msg); + + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(msg.message(), &err, &debug); + updateError(QMediaRecorder::ResourceError, QString::fromUtf8(err.get()->message)); if (!m_finalizing) stop(); finalize(); + return false; } - return false; + case GST_MESSAGE_STATE_CHANGED: { + if constexpr (traceStateChange) + qCDebug(qLcMediaEncoderGst) + << "received state change" << QCompactGstMessageAdaptor(msg); + + return false; + } + + default: + return false; + }; } qint64 QGstreamerMediaEncoder::duration() const @@ -142,13 +120,13 @@ static GstEncodingContainerProfile *createContainerProfile(const QMediaEncoderSe { auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo(); - QGstMutableCaps caps = formatInfo->formatCaps(settings.fileFormat()); + auto caps = formatInfo->formatCaps(settings.fileFormat()); - GstEncodingContainerProfile *profile = (GstEncodingContainerProfile *)gst_encoding_container_profile_new( - "container_profile", - (gchar *)"custom container profile", - const_cast<GstCaps *>(caps.get()), - nullptr); //preset + GstEncodingContainerProfile *profile = + (GstEncodingContainerProfile *)gst_encoding_container_profile_new( + "container_profile", (gchar *)"custom container profile", + const_cast<GstCaps *>(caps.caps()), + nullptr); // preset return profile; } @@ -156,15 +134,18 @@ static GstEncodingProfile *createVideoProfile(const QMediaEncoderSettings &setti { auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo(); - QGstMutableCaps caps = formatInfo->videoCaps(settings.mediaFormat()); + QGstCaps caps = formatInfo->videoCaps(settings.mediaFormat()); if (caps.isNull()) return nullptr; - GstEncodingVideoProfile *profile = gst_encoding_video_profile_new( - const_cast<GstCaps *>(caps.get()), - nullptr, - nullptr, //restriction - 0); //presence + QSize videoResolution = settings.videoResolution(); + if (videoResolution.isValid()) + caps.setResolution(videoResolution); + + GstEncodingVideoProfile *profile = + gst_encoding_video_profile_new(const_cast<GstCaps *>(caps.caps()), nullptr, + nullptr, // restriction + 0); // presence gst_encoding_video_profile_set_pass(profile, 0); gst_encoding_video_profile_set_variableframerate(profile, TRUE); @@ -180,11 +161,11 @@ static GstEncodingProfile *createAudioProfile(const QMediaEncoderSettings &setti if (caps.isNull()) return nullptr; - GstEncodingProfile *profile = (GstEncodingProfile *)gst_encoding_audio_profile_new( - const_cast<GstCaps *>(caps.get()), - nullptr, //preset - nullptr, //restriction - 0); //presence + GstEncodingProfile *profile = + (GstEncodingProfile *)gst_encoding_audio_profile_new(const_cast<GstCaps *>(caps.caps()), + nullptr, // preset + nullptr, // restriction + 0); // presence return profile; } @@ -281,7 +262,7 @@ void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings) const auto hasAudio = m_session->audioInput() != nullptr; if (!hasVideo && !hasAudio) { - error(QMediaRecorder::ResourceError, QMediaRecorder::tr("No camera or audio input")); + updateError(QMediaRecorder::ResourceError, QMediaRecorder::tr("No camera or audio input")); return; } @@ -292,16 +273,18 @@ void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings) auto location = QMediaStorageLocation::generateFileName(outputLocation().toLocalFile(), primaryLocation, container); QUrl actualSink = QUrl::fromLocalFile(QDir::currentPath()).resolved(location); - qCDebug(qLcMediaEncoder) << "recording new video to" << actualSink; + qCDebug(qLcMediaEncoderGst) << "recording new video to" << actualSink; Q_ASSERT(!actualSink.isEmpty()); - gstEncoder = QGstElement("encodebin", "encodebin"); + gstEncoder = QGstBin::createFromFactory("encodebin", "encodebin"); + Q_ASSERT(gstEncoder); auto *encodingProfile = createEncodingProfile(settings); g_object_set (gstEncoder.object(), "profile", encodingProfile, nullptr); gst_encoding_profile_unref(encodingProfile); - gstFileSink = QGstElement("filesink", "filesink"); + gstFileSink = QGstElement::createFromFactory("filesink", "filesink"); + Q_ASSERT(gstFileSink); gstFileSink.set("location", QFile::encodeName(actualSink.toLocalFile()).constData()); gstFileSink.set("async", false); @@ -327,17 +310,19 @@ void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings) videoPauseControl.installOn(videoSink); } - gstPipeline.add(gstEncoder, gstFileSink); - gstEncoder.link(gstFileSink); - m_metaData.setMetaData(gstEncoder.bin()); + capturePipeline.modifyPipelineWhileNotRunning([&] { + capturePipeline.add(gstEncoder, gstFileSink); + qLinkGstElements(gstEncoder, gstFileSink); + applyMetaDataToTagSetter(m_metaData, gstEncoder); - m_session->linkEncoder(audioSink, videoSink); + m_session->linkEncoder(audioSink, videoSink); - gstEncoder.syncStateWithParent(); - gstFileSink.syncStateWithParent(); + gstEncoder.syncStateWithParent(); + gstFileSink.syncStateWithParent(); + }); signalDurationChangedTimer.start(); - gstPipeline.dumpGraph("recording"); + capturePipeline.dumpGraph("recording"); durationChanged(0); stateChanged(QMediaRecorder::RecordingState); @@ -349,13 +334,14 @@ void QGstreamerMediaEncoder::pause() if (!m_session || m_finalizing || state() != QMediaRecorder::RecordingState) return; signalDurationChangedTimer.stop(); - gstPipeline.dumpGraph("before-pause"); + durationChanged(duration()); + capturePipeline.dumpGraph("before-pause"); stateChanged(QMediaRecorder::PausedState); } void QGstreamerMediaEncoder::resume() { - gstPipeline.dumpGraph("before-resume"); + capturePipeline.dumpGraph("before-resume"); if (!m_session || m_finalizing || state() != QMediaRecorder::PausedState) return; signalDurationChangedTimer.start(); @@ -366,12 +352,13 @@ void QGstreamerMediaEncoder::stop() { if (!m_session || m_finalizing || state() == QMediaRecorder::StoppedState) return; - qCDebug(qLcMediaEncoder) << "stop"; + durationChanged(duration()); + qCDebug(qLcMediaEncoderGst) << "stop"; m_finalizing = true; m_session->unlinkEncoder(); signalDurationChangedTimer.stop(); - qCDebug(qLcMediaEncoder) << ">>>>>>>>>>>>> sending EOS"; + qCDebug(qLcMediaEncoderGst) << ">>>>>>>>>>>>> sending EOS"; gstEncoder.sendEos(); } @@ -380,12 +367,9 @@ void QGstreamerMediaEncoder::finalize() if (!m_session || gstEncoder.isNull()) return; - qCDebug(qLcMediaEncoder) << "finalize"; + qCDebug(qLcMediaEncoderGst) << "finalize"; - gstPipeline.remove(gstEncoder); - gstPipeline.remove(gstFileSink); - gstEncoder.setStateSync(GST_STATE_NULL); - gstFileSink.setStateSync(GST_STATE_NULL); + capturePipeline.stopAndRemoveElements(gstEncoder, gstFileSink); gstFileSink = {}; gstEncoder = {}; m_finalizing = false; @@ -396,7 +380,7 @@ void QGstreamerMediaEncoder::setMetaData(const QMediaMetaData &metaData) { if (!m_session) return; - m_metaData = static_cast<const QGstreamerMetaData &>(metaData); + m_metaData = metaData; } QMediaMetaData QGstreamerMediaEncoder::metaData() const @@ -414,19 +398,22 @@ void QGstreamerMediaEncoder::setCaptureSession(QPlatformMediaCaptureSession *ses stop(); if (m_finalizing) { QEventLoop loop; - loop.connect(mediaRecorder(), SIGNAL(recorderStateChanged(RecorderState)), SLOT(quit())); + QObject::connect(mediaRecorder(), &QMediaRecorder::recorderStateChanged, &loop, + &QEventLoop::quit); loop.exec(); } - gstPipeline.removeMessageFilter(this); - gstPipeline = {}; + capturePipeline.removeMessageFilter(this); + capturePipeline = {}; } m_session = captureSession; if (!m_session) return; - gstPipeline = captureSession->gstPipeline; - gstPipeline.set("message-forward", true); - gstPipeline.installMessageFilter(this); + capturePipeline = captureSession->capturePipeline; + capturePipeline.set("message-forward", true); + capturePipeline.installMessageFilter(this); } + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h index 554b79207..56e8c193b 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERENCODERCONTROL_H @@ -52,14 +16,14 @@ // We mean it. // -#include <private/qplatformmediarecorder_p.h> -#include "qgstreamermediacapture_p.h" -#include "qgstreamermetadata_p.h" +#include <mediacapture/qgstreamermediacapture_p.h> +#include <common/qgstreamermetadata_p.h> +#include <QtMultimedia/private/qplatformmediarecorder_p.h> #include <QtCore/qurl.h> #include <QtCore/qdir.h> -#include <qelapsedtimer.h> -#include <qtimer.h> +#include <QtCore/qelapsedtimer.h> +#include <QtCore/qtimer.h> QT_BEGIN_NAMESPACE @@ -69,7 +33,7 @@ class QGstreamerMessage; class QGstreamerMediaEncoder : public QPlatformMediaRecorder, QGstreamerBusMessageFilter { public: - QGstreamerMediaEncoder(QMediaRecorder *parent); + explicit QGstreamerMediaEncoder(QMediaRecorder *parent); virtual ~QGstreamerMediaEncoder(); bool isLocationWritable(const QUrl &sink) const override; @@ -92,7 +56,7 @@ private: private: struct PauseControl { - PauseControl(QPlatformMediaRecorder &encoder) : encoder(encoder) {} + explicit PauseControl(QPlatformMediaRecorder &encoder) : encoder(encoder) { } GstPadProbeReturn processBuffer(QGstPad pad, GstPadProbeInfo *info); void installOn(QGstPad pad); @@ -112,10 +76,10 @@ private: void finalize(); QGstreamerMediaCapture *m_session = nullptr; - QGstreamerMetaData m_metaData; + QMediaMetaData m_metaData; QTimer signalDurationChangedTimer; - QGstPipeline gstPipeline; + QGstPipeline capturePipeline; QGstBin gstEncoder; QGstElement gstFileSink; diff --git a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp index 8c17ed7e7..a657fc52f 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp +++ b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp @@ -1,56 +1,22 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include <common/qglist_helper_p.h> #include "qgstreamerformatinfo_p.h" -#include "qgstutils_p.h" +#include <gst/gst.h> QT_BEGIN_NAMESPACE -QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructure structure) +QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!name || strncmp(name, "audio/", 6)) + if (!name || (strncmp(name, "audio/", 6) != 0)) return QMediaFormat::AudioCodec::Unspecified; name += 6; - if (!strcmp(name, "mpeg")) { + if (name == "mpeg"sv) { auto version = structure["mpegversion"].toInt(); if (version == 1) { auto layer = structure["layer"]; @@ -59,91 +25,120 @@ QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructure s } if (version == 4) return QMediaFormat::AudioCodec::AAC; - } else if (!strcmp(name, "x-ac3")) { + return QMediaFormat::AudioCodec::Unspecified; + } + if (name == "x-ac3"sv) return QMediaFormat::AudioCodec::AC3; - } else if (!strcmp(name, "x-eac3")) { + + if (name == "x-eac3"sv) return QMediaFormat::AudioCodec::EAC3; - } else if (!strcmp(name, "x-flac")) { + + if (name == "x-flac"sv) return QMediaFormat::AudioCodec::FLAC; - } else if (!strcmp(name, "x-alac")) { + + if (name == "x-alac"sv) return QMediaFormat::AudioCodec::ALAC; - } else if (!strcmp(name, "x-true-hd")) { + + if (name == "x-true-hd"sv) return QMediaFormat::AudioCodec::DolbyTrueHD; - } else if (!strcmp(name, "x-vorbis")) { + + if (name == "x-vorbis"sv) return QMediaFormat::AudioCodec::Vorbis; - } else if (!strcmp(name, "x-opus")) { + + if (name == "x-opus"sv) return QMediaFormat::AudioCodec::Opus; - } else if (!strcmp(name, "x-wav")) { + + if (name == "x-wav"sv) return QMediaFormat::AudioCodec::Wave; - } else if (!strcmp(name, "x-wma")) { + + if (name == "x-wma"sv) return QMediaFormat::AudioCodec::WMA; - } + return QMediaFormat::AudioCodec::Unspecified; } -QMediaFormat::VideoCodec QGstreamerFormatInfo::videoCodecForCaps(QGstStructure structure) +QMediaFormat::VideoCodec QGstreamerFormatInfo::videoCodecForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!name || strncmp(name, "video/", 6)) + if (!name || (strncmp(name, "video/", 6) != 0)) return QMediaFormat::VideoCodec::Unspecified; name += 6; - if (!strcmp(name, "mpeg")) { + if (name == "mpeg"sv) { auto version = structure["mpegversion"].toInt(); if (version == 1) return QMediaFormat::VideoCodec::MPEG1; - else if (version == 2) + if (version == 2) return QMediaFormat::VideoCodec::MPEG2; - else if (version == 4) + if (version == 4) return QMediaFormat::VideoCodec::MPEG4; - } else if (!strcmp(name, "x-h264")) { + return QMediaFormat::VideoCodec::Unspecified; + } + if (name == "x-h264"sv) return QMediaFormat::VideoCodec::H264; + #if GST_CHECK_VERSION(1, 17, 0) // x265enc seems to be broken on 1.16 at least - } else if (!strcmp(name, "x-h265")) { + if (name == "x-h265"sv) return QMediaFormat::VideoCodec::H265; #endif - } else if (!strcmp(name, "x-vp8")) { + + if (name == "x-vp8"sv) return QMediaFormat::VideoCodec::VP8; - } else if (!strcmp(name, "x-vp9")) { + + if (name == "x-vp9"sv) return QMediaFormat::VideoCodec::VP9; - } else if (!strcmp(name, "x-av1")) { + + if (name == "x-av1"sv) return QMediaFormat::VideoCodec::AV1; - } else if (!strcmp(name, "x-theora")) { + + if (name == "x-theora"sv) return QMediaFormat::VideoCodec::Theora; - } else if (!strcmp(name, "x-jpeg")) { + + if (name == "x-jpeg"sv) return QMediaFormat::VideoCodec::MotionJPEG; - } else if (!strcmp(name, "x-wmv")) { + + if (name == "x-wmv"sv) return QMediaFormat::VideoCodec::WMV; - } + return QMediaFormat::VideoCodec::Unspecified; } -QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructure structure) +QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!strcmp(name, "video/x-ms-asf")) { + if (name == "video/x-ms-asf"sv) return QMediaFormat::FileFormat::WMV; - } else if (!strcmp(name, "video/x-msvideo")) { + + if (name == "video/x-msvideo"sv) return QMediaFormat::FileFormat::AVI; - } else if (!strcmp(name, "video/x-matroska")) { + + if (name == "video/x-matroska"sv) return QMediaFormat::FileFormat::Matroska; - } else if (!strcmp(name, "video/quicktime")) { - auto variant = structure["variant"].toString(); + + if (name == "video/quicktime"sv) { + const char *variant = structure["variant"].toString(); if (!variant) return QMediaFormat::FileFormat::QuickTime; - else if (!strcmp(variant, "iso")) + if (variant == "iso"sv) return QMediaFormat::FileFormat::MPEG4; - } else if (!strcmp(name, "video/ogg")) { + } + if (name == "video/ogg"sv) return QMediaFormat::FileFormat::Ogg; - } else if (!strcmp(name, "video/webm")) { + + if (name == "video/webm"sv) return QMediaFormat::FileFormat::WebM; - } else if (!strcmp(name, "audio/x-m4a")) { + + if (name == "audio/x-m4a"sv) return QMediaFormat::FileFormat::Mpeg4Audio; - } else if (!strcmp(name, "audio/x-wav")) { + + if (name == "audio/x-wav"sv) return QMediaFormat::FileFormat::Wave; - } else if (!strcmp(name, "audio/mpeg")) { + + if (name == "audio/mpeg"sv) { auto mpegversion = structure["mpegversion"].toInt(); if (mpegversion == 1) { auto layer = structure["layer"]; @@ -151,23 +146,28 @@ QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructure s return QMediaFormat::FileFormat::MP3; } } + return QMediaFormat::UnspecifiedFormat; } -QImageCapture::FileFormat QGstreamerFormatInfo::imageFormatForCaps(QGstStructure structure) +QImageCapture::FileFormat QGstreamerFormatInfo::imageFormatForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!strcmp(name, "image/jpeg")) { + if (name == "image/jpeg"sv) return QImageCapture::JPEG; - } else if (!strcmp(name, "image/png")) { + + if (name == "image/png"sv) return QImageCapture::PNG; - } else if (!strcmp(name, "image/webp")) { + + if (name == "image/webp"sv) return QImageCapture::WebP; - } else if (!strcmp(name, "image/tiff")) { + + if (name == "image/tiff"sv) return QImageCapture::Tiff; - } + return QImageCapture::UnspecifiedFormat; } @@ -181,21 +181,16 @@ static QPair<QList<QMediaFormat::AudioCodec>, QList<QMediaFormat::VideoCodec>> g GList *elementList = gst_element_factory_list_get_elements(decode ? GST_ELEMENT_FACTORY_TYPE_DECODER : GST_ELEMENT_FACTORY_TYPE_ENCODER, GST_RANK_MARGINAL); - GList *element = elementList; - while (element) { - GstElementFactory *factory = (GstElementFactory *)element->data; - element = element->next; - - const GList *padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; - + for (GstElementFactory *factory : + QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { if (padTemplate->direction == padDirection) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); auto a = QGstreamerFormatInfo::audioCodecForCaps(structure); if (a != QMediaFormat::AudioCodec::Unspecified && !audio.contains(a)) audio.append(a); @@ -219,25 +214,23 @@ QList<QGstreamerFormatInfo::CodecMap> QGstreamerFormatInfo::getMuxerList(bool de GstPadDirection padDirection = demuxer ? GST_PAD_SINK : GST_PAD_SRC; - GList *elementList = gst_element_factory_list_get_elements(demuxer ? GST_ELEMENT_FACTORY_TYPE_DEMUXER : GST_ELEMENT_FACTORY_TYPE_MUXER, - GST_RANK_MARGINAL); - GList *element = elementList; - while (element) { - GstElementFactory *factory = (GstElementFactory *)element->data; - element = element->next; + GList *elementList = gst_element_factory_list_get_elements( + demuxer ? GST_ELEMENT_FACTORY_TYPE_DEMUXER : GST_ELEMENT_FACTORY_TYPE_MUXER, + GST_RANK_MARGINAL); + for (GstElementFactory *factory : + QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { QList<QMediaFormat::FileFormat> fileFormats; - const GList *padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { if (padTemplate->direction == padDirection) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); auto fmt = fileFormatForCaps(structure); if (fmt != QMediaFormat::UnspecifiedFormat) fileFormats.append(fmt); @@ -250,18 +243,17 @@ QList<QGstreamerFormatInfo::CodecMap> QGstreamerFormatInfo::getMuxerList(bool de QList<QMediaFormat::AudioCodec> audioCodecs; QList<QMediaFormat::VideoCodec> videoCodecs; - padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { // check the other side for supported inputs/outputs if (padTemplate->direction != padDirection) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); bool acceptsRawAudio = false; for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); if (structure.name() == "audio/x-raw") acceptsRawAudio = true; auto audio = audioCodecForCaps(structure); @@ -290,7 +282,7 @@ QList<QGstreamerFormatInfo::CodecMap> QGstreamerFormatInfo::getMuxerList(bool de } } if (!audioCodecs.isEmpty() || !videoCodecs.isEmpty()) { - for (auto f : qAsConst(fileFormats)) { + for (auto f : std::as_const(fileFormats)) { muxers.append({f, audioCodecs, videoCodecs}); if (f == QMediaFormat::MPEG4 && !fileFormats.contains(QMediaFormat::Mpeg4Audio)) { muxers.append({QMediaFormat::Mpeg4Audio, audioCodecs, {}}); @@ -313,21 +305,17 @@ static QList<QImageCapture::FileFormat> getImageFormatList() GList *elementList = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER, GST_RANK_MARGINAL); - GList *element = elementList; - while (element) { - GstElementFactory *factory = (GstElementFactory *)element->data; - element = element->next; - - const GList *padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; + for (GstElementFactory *factory : + QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { if (padTemplate->direction == GST_PAD_SRC) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + QGstCaps caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); auto f = QGstreamerFormatInfo::imageFormatForCaps(structure); if (f != QImageCapture::UnspecifiedFormat) { // qDebug() << structure.toString() << f; @@ -387,7 +375,7 @@ QGstreamerFormatInfo::QGstreamerFormatInfo() QGstreamerFormatInfo::~QGstreamerFormatInfo() = default; -QGstMutableCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const +QGstCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const { auto format = f.fileFormat(); Q_ASSERT(format != QMediaFormat::UnspecifiedFormat); @@ -407,14 +395,14 @@ QGstMutableCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const "audio/x-flac", // FLAC "audio/x-wav" // Wave }; - return gst_caps_from_string(capsForFormat[format]); + return QGstCaps(gst_caps_from_string(capsForFormat[format]), QGstCaps::HasRef); } -QGstMutableCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const +QGstCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const { auto codec = f.audioCodec(); if (codec == QMediaFormat::AudioCodec::Unspecified) - return nullptr; + return {}; const char *capsForCodec[(int)QMediaFormat::AudioCodec::LastAudioCodec + 1] = { "audio/mpeg, mpegversion=(int)1, layer=(int)3", // MP3 @@ -429,14 +417,14 @@ QGstMutableCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const "audio/x-wma", // WMA "audio/x-alac", // ALAC }; - return gst_caps_from_string(capsForCodec[(int)codec]); + return QGstCaps(gst_caps_from_string(capsForCodec[(int)codec]), QGstCaps::HasRef); } -QGstMutableCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const +QGstCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const { auto codec = f.videoCodec(); if (codec == QMediaFormat::VideoCodec::Unspecified) - return nullptr; + return {}; const char *capsForCodec[(int)QMediaFormat::VideoCodec::LastVideoCodec + 1] = { "video/mpeg, mpegversion=(int)1", // MPEG1, @@ -451,7 +439,7 @@ QGstMutableCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const "audio/x-wmv", // WMV "video/x-jpeg", // MotionJPEG, }; - return gst_caps_from_string(capsForCodec[(int)codec]); + return QGstCaps(gst_caps_from_string(capsForCodec[(int)codec]), QGstCaps::HasRef); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h index 3cfea9dc5..bba10edb9 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h +++ b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERFORMATINFO_H #define QGSTREAMERFORMATINFO_H @@ -52,9 +16,8 @@ // #include <private/qplatformmediaformatinfo_p.h> -#include <qhash.h> #include <qlist.h> -#include <qgstutils_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE @@ -64,14 +27,14 @@ public: QGstreamerFormatInfo(); ~QGstreamerFormatInfo(); - QGstMutableCaps formatCaps(const QMediaFormat &f) const; - QGstMutableCaps audioCaps(const QMediaFormat &f) const; - QGstMutableCaps videoCaps(const QMediaFormat &f) const; + QGstCaps formatCaps(const QMediaFormat &f) const; + QGstCaps audioCaps(const QMediaFormat &f) const; + QGstCaps videoCaps(const QMediaFormat &f) const; - static QMediaFormat::AudioCodec audioCodecForCaps(QGstStructure structure); - static QMediaFormat::VideoCodec videoCodecForCaps(QGstStructure structure); - static QMediaFormat::FileFormat fileFormatForCaps(QGstStructure structure); - static QImageCapture::FileFormat imageFormatForCaps(QGstStructure structure); + static QMediaFormat::AudioCodec audioCodecForCaps(QGstStructureView structure); + static QMediaFormat::VideoCodec videoCodecForCaps(QGstStructureView structure); + static QMediaFormat::FileFormat fileFormatForCaps(QGstStructureView structure); + static QImageCapture::FileFormat imageFormatForCaps(QGstStructureView structure); QList<CodecMap> getMuxerList(bool demuxer, QList<QMediaFormat::AudioCodec> audioCodecs, QList<QMediaFormat::VideoCodec> videoCodecs); }; diff --git a/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp b/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp index 8ed2fda03..87c514f2e 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp +++ b/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp @@ -1,150 +1,242 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamerintegration_p.h" -#include "qgstreamervideodevices_p.h" -#include "qgstreamermediaplayer_p.h" -#include "qgstreamermediacapture_p.h" -#include "qgstreameraudiodecoder_p.h" -#include "qgstreamercamera_p.h" -#include "qgstreamermediaencoder_p.h" -#include "qgstreamerimagecapture_p.h" -#include "qgstreamerformatinfo_p.h" -#include "qgstreamervideosink_p.h" -#include "qgstreameraudioinput_p.h" -#include "qgstreameraudiooutput_p.h" -#include <QtMultimedia/private/qplatformmediaplugin_p.h> - -#include <memory> +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <qgstreamerintegration_p.h> +#include <qgstreamerformatinfo_p.h> +#include <qgstreamervideodevices_p.h> +#include <audio/qgstreameraudiodevice_p.h> +#include <audio/qgstreameraudiodecoder_p.h> +#include <common/qgstreameraudioinput_p.h> +#include <common/qgstreameraudiooutput_p.h> +#include <common/qgstreamermediaplayer_p.h> +#include <common/qgstreamervideosink_p.h> +#include <mediacapture/qgstreamercamera_p.h> +#include <mediacapture/qgstreamerimagecapture_p.h> +#include <mediacapture/qgstreamermediacapture_p.h> +#include <mediacapture/qgstreamermediaencoder_p.h> + +#include <QtCore/qloggingcategory.h> +#include <QtMultimedia/private/qmediaplayer_p.h> +#include <QtMultimedia/private/qmediacapturesession_p.h> +#include <QtMultimedia/private/qcameradevice_p.h> QT_BEGIN_NAMESPACE -class QGstreamerMediaPlugin : public QPlatformMediaPlugin +static thread_local bool inCustomCameraConstruction = false; +static thread_local QGstElement pendingCameraElement{}; + +QGStreamerPlatformSpecificInterfaceImplementation:: + ~QGStreamerPlatformSpecificInterfaceImplementation() = default; + +QAudioDevice QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerAudioInput( + const QByteArray &gstreamerPipeline) +{ + return qMakeCustomGStreamerAudioInput(gstreamerPipeline); +} + +QAudioDevice QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerAudioOutput( + const QByteArray &gstreamerPipeline) +{ + return qMakeCustomGStreamerAudioOutput(gstreamerPipeline); +} + +QCamera *QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerCamera( + const QByteArray &gstreamerPipeline, QObject *parent) { - Q_OBJECT - Q_PLUGIN_METADATA(IID QPlatformMediaPlugin_iid FILE "gstreamer.json") + QCameraDevicePrivate *info = new QCameraDevicePrivate; + info->id = gstreamerPipeline; + QCameraDevice device = info->create(); + + inCustomCameraConstruction = true; + auto guard = qScopeGuard([] { + inCustomCameraConstruction = false; + }); -public: - QGstreamerMediaPlugin() - : QPlatformMediaPlugin() - {} + return new QCamera(device, parent); +} - QPlatformMediaIntegration* create(const QString &name) override - { - if (name == QLatin1String("gstreamer")) - return new QGstreamerIntegration; +QCamera * +QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerCamera(GstElement *element, + QObject *parent) +{ + QCameraDevicePrivate *info = new QCameraDevicePrivate; + info->id = "Custom Camera from GstElement"; + QCameraDevice device = info->create(); + + pendingCameraElement = QGstElement{ + element, + QGstElement::NeedsRef, + }; + + inCustomCameraConstruction = true; + auto guard = qScopeGuard([] { + inCustomCameraConstruction = false; + Q_ASSERT(!pendingCameraElement); + }); + + return new QCamera(device, parent); +} + +GstPipeline *QGStreamerPlatformSpecificInterfaceImplementation::gstPipeline(QMediaPlayer *player) +{ + auto *priv = reinterpret_cast<QMediaPlayerPrivate *>(QMediaPlayerPrivate::get(player)); + if (!priv) return nullptr; - } + + QGstreamerMediaPlayer *gstreamerPlayer = dynamic_cast<QGstreamerMediaPlayer *>(priv->control); + return gstreamerPlayer ? gstreamerPlayer->pipeline().pipeline() : nullptr; +} + +GstPipeline * +QGStreamerPlatformSpecificInterfaceImplementation::gstPipeline(QMediaCaptureSession *session) +{ + auto *priv = QMediaCaptureSessionPrivate::get(session); + if (!priv) + return nullptr; + + QGstreamerMediaCapture *gstreamerCapture = + dynamic_cast<QGstreamerMediaCapture *>(priv->captureSession.get()); + return gstreamerCapture ? gstreamerCapture->pipeline().pipeline() : nullptr; +} + +Q_LOGGING_CATEGORY(lcGstreamer, "qt.multimedia.gstreamer") + +namespace { + +void rankDownPlugin(GstRegistry *reg, const char *name) +{ + QGstPluginFeatureHandle pluginFeature{ + gst_registry_lookup_feature(reg, name), + QGstPluginFeatureHandle::HasRef, + }; + if (pluginFeature) + gst_plugin_feature_set_rank(pluginFeature.get(), GST_RANK_PRIMARY - 1); +} + +// https://gstreamer.freedesktop.org/documentation/vaapi/index.html +constexpr auto vaapiPluginNames = { + "vaapidecodebin", "vaapih264dec", "vaapih264enc", "vaapih265dec", + "vaapijpegdec", "vaapijpegenc", "vaapimpeg2dec", "vaapipostproc", + "vaapisink", "vaapivp8dec", "vaapivp9dec", +}; + +// https://gstreamer.freedesktop.org/documentation/va/index.html +constexpr auto vaPluginNames = { + "vaav1dec", "vacompositor", "vadeinterlace", "vah264dec", "vah264enc", "vah265dec", + "vajpegdec", "vampeg2dec", "vapostproc", "vavp8dec", "vavp9dec", +}; + +// https://gstreamer.freedesktop.org/documentation/nvcodec/index.html +constexpr auto nvcodecPluginNames = { + "cudaconvert", "cudaconvertscale", "cudadownload", "cudaipcsink", "cudaipcsrc", + "cudascale", "cudaupload", "nvautogpuh264enc", "nvautogpuh265enc", "nvav1dec", + "nvcudah264enc", "nvcudah265enc", "nvd3d11h264enc", "nvd3d11h265enc", "nvh264dec", + "nvh264enc", "nvh265dec", "nvh265enc", "nvjpegdec", "nvjpegenc", + "nvmpeg2videodec", "nvmpeg4videodec", "nvmpegvideodec", "nvvp8dec", "nvvp9dec", }; +} // namespace + QGstreamerIntegration::QGstreamerIntegration() + : QPlatformMediaIntegration(QLatin1String("gstreamer")) { gst_init(nullptr, nullptr); - m_videoDevices = new QGstreamerVideoDevices(this); - m_formatsInfo = new QGstreamerFormatInfo(); + qCDebug(lcGstreamer) << "Using gstreamer version: " << gst_version_string(); + + GstRegistry *reg = gst_registry_get(); + + if constexpr (!GST_CHECK_VERSION(1, 22, 0)) { + GstRegistry* reg = gst_registry_get(); + for (const char *name : vaapiPluginNames) + rankDownPlugin(reg, name); + } + + if (qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_VA")) { + for (const char *name : vaPluginNames) + rankDownPlugin(reg, name); + } + + if (qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_NVCODEC")) { + for (const char *name : nvcodecPluginNames) + rankDownPlugin(reg, name); + } } -QGstreamerIntegration::~QGstreamerIntegration() +QPlatformMediaFormatInfo *QGstreamerIntegration::createFormatInfo() { - delete m_formatsInfo; + return new QGstreamerFormatInfo(); } -QPlatformMediaFormatInfo *QGstreamerIntegration::formatInfo() +QPlatformVideoDevices *QGstreamerIntegration::createVideoDevices() { - return m_formatsInfo; + return new QGstreamerVideoDevices(this); } -const QGstreamerFormatInfo *QGstreamerIntegration::gstFormatsInfo() const +const QGstreamerFormatInfo *QGstreamerIntegration::gstFormatsInfo() { - return m_formatsInfo; + return static_cast<const QGstreamerFormatInfo *>(formatInfo()); } -QPlatformAudioDecoder *QGstreamerIntegration::createAudioDecoder(QAudioDecoder *decoder) +QMaybe<QPlatformAudioDecoder *> QGstreamerIntegration::createAudioDecoder(QAudioDecoder *decoder) { - return new QGstreamerAudioDecoder(decoder); + return QGstreamerAudioDecoder::create(decoder); } -QPlatformMediaCaptureSession *QGstreamerIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QGstreamerIntegration::createCaptureSession() { - return new QGstreamerMediaCapture(); + return QGstreamerMediaCapture::create(); } -QPlatformMediaPlayer *QGstreamerIntegration::createPlayer(QMediaPlayer *player) +QMaybe<QPlatformMediaPlayer *> QGstreamerIntegration::createPlayer(QMediaPlayer *player) { - return new QGstreamerMediaPlayer(player); + return QGstreamerMediaPlayer::create(player); } -QPlatformCamera *QGstreamerIntegration::createCamera(QCamera *camera) +QMaybe<QPlatformCamera *> QGstreamerIntegration::createCamera(QCamera *camera) { - return new QGstreamerCamera(camera); + if (inCustomCameraConstruction) { + QGstElement element = std::exchange(pendingCameraElement, {}); + return element ? new QGstreamerCustomCamera{ camera, std::move(element) } + : new QGstreamerCustomCamera{ camera }; + } + + return QGstreamerCamera::create(camera); } -QPlatformMediaRecorder *QGstreamerIntegration::createRecorder(QMediaRecorder *recorder) +QMaybe<QPlatformMediaRecorder *> QGstreamerIntegration::createRecorder(QMediaRecorder *recorder) { return new QGstreamerMediaEncoder(recorder); } -QPlatformImageCapture *QGstreamerIntegration::createImageCapture(QImageCapture *imageCapture) +QMaybe<QPlatformImageCapture *> QGstreamerIntegration::createImageCapture(QImageCapture *imageCapture) { - return new QGstreamerImageCapture(imageCapture); + return QGstreamerImageCapture::create(imageCapture); } -QPlatformVideoSink *QGstreamerIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QGstreamerIntegration::createVideoSink(QVideoSink *sink) { return new QGstreamerVideoSink(sink); } -QPlatformAudioInput *QGstreamerIntegration::createAudioInput(QAudioInput *q) +QMaybe<QPlatformAudioInput *> QGstreamerIntegration::createAudioInput(QAudioInput *q) { - return new QGstreamerAudioInput(q); + return QGstreamerAudioInput::create(q); } -QPlatformAudioOutput *QGstreamerIntegration::createAudioOutput(QAudioOutput *q) +QMaybe<QPlatformAudioOutput *> QGstreamerIntegration::createAudioOutput(QAudioOutput *q) { - return new QGstreamerAudioOutput(q); + return QGstreamerAudioOutput::create(q); } -GstDevice *QGstreamerIntegration::videoDevice(const QByteArray &id) const +GstDevice *QGstreamerIntegration::videoDevice(const QByteArray &id) { - return m_videoDevices ? static_cast<QGstreamerVideoDevices*>(m_videoDevices)->videoDevice(id) : nullptr; + const auto devices = videoDevices(); + return devices ? static_cast<QGstreamerVideoDevices *>(devices)->videoDevice(id) : nullptr; } -QT_END_NAMESPACE +QAbstractPlatformSpecificInterface *QGstreamerIntegration::platformSpecificInterface() +{ + return &m_platformSpecificImplementation; +} -#include "qgstreamerintegration.moc" +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h b/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h index 4b8d02efd..229bbd48e 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h +++ b/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERINTEGRATION_H #define QGSTREAMERINTEGRATION_H @@ -51,39 +15,63 @@ // We mean it. // -#include <private/qplatformmediaintegration_p.h> +#include <QtMultimedia/private/qplatformmediaintegration_p.h> +#include <QtMultimedia/private/qgstreamer_platformspecificinterface_p.h> + #include <gst/gst.h> QT_BEGIN_NAMESPACE class QGstreamerFormatInfo; +class QGStreamerPlatformSpecificInterfaceImplementation : public QGStreamerPlatformSpecificInterface +{ +public: + ~QGStreamerPlatformSpecificInterfaceImplementation() override; + + QAudioDevice makeCustomGStreamerAudioInput(const QByteArray &gstreamerPipeline) override; + QAudioDevice makeCustomGStreamerAudioOutput(const QByteArray &gstreamerPipeline) override; + QCamera *makeCustomGStreamerCamera(const QByteArray &gstreamerPipeline, + QObject *parent) override; + + QCamera *makeCustomGStreamerCamera(GstElement *, QObject *parent) override; + + GstPipeline *gstPipeline(QMediaPlayer *) override; + GstPipeline *gstPipeline(QMediaCaptureSession *) override; +}; + class QGstreamerIntegration : public QPlatformMediaIntegration { public: QGstreamerIntegration(); - ~QGstreamerIntegration(); - static QGstreamerIntegration *instance() { return static_cast<QGstreamerIntegration *>(QPlatformMediaIntegration::instance()); } - QPlatformMediaFormatInfo *formatInfo() override; + static QGstreamerIntegration *instance() + { + return static_cast<QGstreamerIntegration *>(QPlatformMediaIntegration::instance()); + } + + QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *decoder) override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *) override; + + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *sink) override; - QPlatformAudioDecoder *createAudioDecoder(QAudioDecoder *decoder) override; - QPlatformMediaCaptureSession *createCaptureSession() override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *player) override; - QPlatformCamera *createCamera(QCamera *) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *) override; - QPlatformImageCapture *createImageCapture(QImageCapture *) override; + QMaybe<QPlatformAudioInput *> createAudioInput(QAudioInput *) override; + QMaybe<QPlatformAudioOutput *> createAudioOutput(QAudioOutput *) override; - QPlatformVideoSink *createVideoSink(QVideoSink *sink) override; + const QGstreamerFormatInfo *gstFormatsInfo(); + GstDevice *videoDevice(const QByteArray &id); - QPlatformAudioInput *createAudioInput(QAudioInput *) override; - QPlatformAudioOutput *createAudioOutput(QAudioOutput *) override; + QAbstractPlatformSpecificInterface *platformSpecificInterface() override; - const QGstreamerFormatInfo *gstFormatsInfo() const; - GstDevice *videoDevice(const QByteArray &id) const; +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; + QPlatformVideoDevices *createVideoDevices() override; -private: - QGstreamerFormatInfo *m_formatsInfo; + QGStreamerPlatformSpecificInterfaceImplementation m_platformSpecificImplementation; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamerplugin.cpp b/src/plugins/multimedia/gstreamer/qgstreamerplugin.cpp new file mode 100644 index 000000000..66ad7f712 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/qgstreamerplugin.cpp @@ -0,0 +1,28 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qplatformmediaplugin_p.h> + +#include <qgstreamerintegration_p.h> + +QT_BEGIN_NAMESPACE + +class QGstreamerMediaPlugin : public QPlatformMediaPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID QPlatformMediaPlugin_iid FILE "gstreamer.json") + +public: + QGstreamerMediaPlugin() = default; + + QPlatformMediaIntegration* create(const QString &name) override + { + if (name == u"gstreamer") + return new QGstreamerIntegration; + return nullptr; + } +}; + +QT_END_NAMESPACE + +#include "qgstreamerplugin.moc" diff --git a/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp b/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp index e7595b4cc..78ac16eb4 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp +++ b/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp @@ -1,201 +1,158 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamervideodevices_p.h" -#include "qmediadevices.h" -#include "private/qcameradevice_p.h" +#include <QtMultimedia/qmediadevices.h> +#include <QtMultimedia/private/qcameradevice_p.h> -#include "qgstreameraudiosource_p.h" -#include "qgstreameraudiosink_p.h" -#include "qgstreameraudiodevice_p.h" -#include "qgstutils_p.h" +#include <common/qgst_p.h> +#include <common/qgstutils_p.h> +#include <common/qglist_helper_p.h> QT_BEGIN_NAMESPACE -static gboolean deviceMonitor(GstBus *, GstMessage *message, gpointer m) +static gboolean deviceMonitorCallback(GstBus *, GstMessage *message, gpointer m) { auto *manager = static_cast<QGstreamerVideoDevices *>(m); - GstDevice *device = nullptr; + QGstDeviceHandle device; - switch (GST_MESSAGE_TYPE (message)) { + switch (GST_MESSAGE_TYPE(message)) { case GST_MESSAGE_DEVICE_ADDED: gst_message_parse_device_added(message, &device); - manager->addDevice(device); + manager->addDevice(std::move(device)); break; case GST_MESSAGE_DEVICE_REMOVED: gst_message_parse_device_removed(message, &device); - manager->removeDevice(device); + manager->removeDevice(std::move(device)); break; default: break; } - if (device) - gst_object_unref (device); return G_SOURCE_CONTINUE; } QGstreamerVideoDevices::QGstreamerVideoDevices(QPlatformMediaIntegration *integration) - : QPlatformVideoDevices(integration) + : QPlatformVideoDevices(integration), + m_deviceMonitor{ + gst_device_monitor_new(), + } { - GstDeviceMonitor *monitor; - GstBus *bus; + gst_device_monitor_add_filter(m_deviceMonitor.get(), "Video/Source", nullptr); - monitor = gst_device_monitor_new(); + QGstBusHandle bus{ + gst_device_monitor_get_bus(m_deviceMonitor.get()), + }; + gst_bus_add_watch(bus.get(), deviceMonitorCallback, this); + gst_device_monitor_start(m_deviceMonitor.get()); - gst_device_monitor_add_filter (monitor, nullptr, nullptr); + GList *devices = gst_device_monitor_get_devices(m_deviceMonitor.get()); - bus = gst_device_monitor_get_bus(monitor); - gst_bus_add_watch(bus, deviceMonitor, this); - gst_object_unref(bus); - - gst_device_monitor_start(monitor); + for (GstDevice *device : QGstUtils::GListRangeAdaptor<GstDevice *>(devices)) { + addDevice(QGstDeviceHandle{ + device, + QGstDeviceHandle::HasRef, + }); + } - auto devices = gst_device_monitor_get_devices(monitor); + g_list_free(devices); +} - while (devices) { - GstDevice *device = static_cast<GstDevice *>(devices->data); - addDevice(device); - gst_object_unref(device); - devices = g_list_delete_link(devices, devices); - } +QGstreamerVideoDevices::~QGstreamerVideoDevices() +{ + gst_device_monitor_stop(m_deviceMonitor.get()); } QList<QCameraDevice> QGstreamerVideoDevices::videoDevices() const { QList<QCameraDevice> devices; - for (auto *d : qAsConst(m_videoSources)) { - QGstStructure properties = gst_device_get_properties(d); - if (!properties.isNull()) { - QCameraDevicePrivate *info = new QCameraDevicePrivate; - auto *desc = gst_device_get_display_name(d); - info->description = QString::fromUtf8(desc); - g_free(desc); - - info->id = properties["device.path"].toString(); - auto def = properties["is-default"].toBool(); + for (const auto &device : m_videoSources) { + QCameraDevicePrivate *info = new QCameraDevicePrivate; + + QGString desc{ + gst_device_get_display_name(device.gstDevice.get()), + }; + info->description = desc.toQString(); + info->id = device.id; + + QUniqueGstStructureHandle properties{ + gst_device_get_properties(device.gstDevice.get()), + }; + if (properties) { + QGstStructureView view{ properties }; + auto def = view["is-default"].toBool(); info->isDefault = def && *def; - if (def) - devices.prepend(info->create()); - else - devices.append(info->create()); - properties.free(); - QGstCaps caps = gst_device_get_caps(d); - if (!caps.isNull()) { - QList<QCameraFormat> formats; - QSet<QSize> photoResolutions; - - int size = caps.size(); - for (int i = 0; i < size; ++i) { - auto cap = caps.at(i); - - QSize resolution = cap.resolution(); - if (!resolution.isValid()) - continue; - - auto pixelFormat = cap.pixelFormat(); - auto frameRate = cap.frameRateRange(); - - auto *f = new QCameraFormatPrivate{ - QSharedData(), - pixelFormat, - resolution, - frameRate.min, - frameRate.max - }; - formats << f->create(); - photoResolutions.insert(resolution); - } - info->videoFormats = formats; - // ### sort resolutions? - info->photoResolutions = photoResolutions.values(); + } + + if (info->isDefault) + devices.prepend(info->create()); + else + devices.append(info->create()); + + auto caps = QGstCaps(gst_device_get_caps(device.gstDevice.get()), QGstCaps::HasRef); + if (!caps.isNull()) { + QList<QCameraFormat> formats; + QSet<QSize> photoResolutions; + + int size = caps.size(); + for (int i = 0; i < size; ++i) { + auto cap = caps.at(i); + + QSize resolution = cap.resolution(); + if (!resolution.isValid()) + continue; + + auto pixelFormat = cap.pixelFormat(); + auto frameRate = cap.frameRateRange(); + + auto *f = new QCameraFormatPrivate{ QSharedData(), pixelFormat, resolution, + frameRate.min, frameRate.max }; + formats << f->create(); + photoResolutions.insert(resolution); } + info->videoFormats = formats; + // ### sort resolutions? + info->photoResolutions = photoResolutions.values(); } } return devices; } -void QGstreamerVideoDevices::addDevice(GstDevice *device) +void QGstreamerVideoDevices::addDevice(QGstDeviceHandle device) { - gchar *type = gst_device_get_device_class(device); -// qDebug() << "adding device:" << device << type << gst_device_get_display_name(device) << gst_structure_to_string(gst_device_get_properties(device)); - gst_object_ref(device); - if (!strcmp(type, "Video/Source") || !strcmp(type, "Source/Video")) { - m_videoSources.insert(device); - videoInputsChanged(); - } else { - gst_object_unref(device); - } - g_free(type); -} + Q_ASSERT(gst_device_has_classes(device.get(), "Video/Source")); -void QGstreamerVideoDevices::removeDevice(GstDevice *device) -{ -// qDebug() << "removing device:" << device << gst_device_get_display_name(device); - if (m_videoSources.remove(device)) - videoInputsChanged(); + auto it = std::find_if(m_videoSources.begin(), m_videoSources.end(), + [&](const QGstRecordDevice &a) { return a.gstDevice == device; }); + + if (it != m_videoSources.end()) + return; - gst_object_unref(device); + m_videoSources.push_back(QGstRecordDevice{ + std::move(device), + QByteArray::number(m_idGenerator), + }); + emit videoInputsChanged(); + m_idGenerator++; } -static GstDevice *getDevice(const QSet<GstDevice *> &devices, const char *key, const QByteArray &id) +void QGstreamerVideoDevices::removeDevice(QGstDeviceHandle device) { - GstDevice *gstDevice = nullptr; - for (auto *d : devices) { - QGstStructure properties = gst_device_get_properties(d); - if (!properties.isNull()) { - auto *name = properties[key].toString(); - if (id == name) { - gstDevice = d; - } - } - properties.free(); - if (gstDevice) - break; + auto it = std::find_if(m_videoSources.begin(), m_videoSources.end(), + [&](const QGstRecordDevice &a) { return a.gstDevice == device; }); + + if (it != m_videoSources.end()) { + m_videoSources.erase(it); + emit videoInputsChanged(); } - return gstDevice; } GstDevice *QGstreamerVideoDevices::videoDevice(const QByteArray &id) const { - return getDevice(m_videoSources, "device.path", id); + auto it = std::find_if(m_videoSources.begin(), m_videoSources.end(), + [&](const QGstRecordDevice &a) { return a.id == id; }); + return it != m_videoSources.end() ? it->gstDevice.get() : nullptr; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h b/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h index cd0acc053..a321ae66b 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h +++ b/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMEDIADEVICES_H #define QGSTREAMERMEDIADEVICES_H @@ -53,24 +17,36 @@ #include <private/qplatformvideodevices_p.h> #include <gst/gst.h> -#include <qset.h> #include <qaudiodevice.h> +#include <vector> + +#include <common/qgst_handle_types_p.h> QT_BEGIN_NAMESPACE class QGstreamerVideoDevices : public QPlatformVideoDevices { public: - QGstreamerVideoDevices(QPlatformMediaIntegration *integration); + explicit QGstreamerVideoDevices(QPlatformMediaIntegration *integration); + ~QGstreamerVideoDevices(); QList<QCameraDevice> videoDevices() const override; GstDevice *videoDevice(const QByteArray &id) const; - void addDevice(GstDevice *); - void removeDevice(GstDevice *); + void addDevice(QGstDeviceHandle); + void removeDevice(QGstDeviceHandle); private: - QSet<GstDevice *> m_videoSources; + struct QGstRecordDevice + { + QGstDeviceHandle gstDevice; + QByteArray id; + }; + + quint64 m_idGenerator = 0; + std::vector<QGstRecordDevice> m_videoSources; + + QGstDeviceMonitorHandle m_deviceMonitor; }; QT_END_NAMESPACE |