diff options
Diffstat (limited to 'src/plugins/multimedia/gstreamer/common')
28 files changed, 2155 insertions, 1215 deletions
diff --git a/src/plugins/multimedia/gstreamer/common/qgst.cpp b/src/plugins/multimedia/gstreamer/common/qgst.cpp index 83d95a9e2..6cf133d6c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgst.cpp @@ -127,11 +127,11 @@ std::optional<QGRange<int>> QGValue::toIntRange() const return QGRange<int>{ gst_value_get_int_range_min(value), gst_value_get_int_range_max(value) }; } -QGstStructure QGValue::toStructure() const +QGstStructureView QGValue::toStructure() const { if (!value || !GST_VALUE_HOLDS_STRUCTURE(value)) - return QGstStructure(); - return QGstStructure(gst_value_get_structure(value)); + return QGstStructureView(nullptr); + return QGstStructureView(gst_value_get_structure(value)); } QGstCaps QGValue::toCaps() const @@ -156,38 +156,52 @@ QGValue QGValue::at(int index) const return QGValue{ gst_value_list_get_value(value, index) }; } -// QGstStructure +// QGstStructureView -QGstStructure::QGstStructure(const GstStructure *s) : structure(s) { } +QGstStructureView::QGstStructureView(const GstStructure *s) : structure(s) { } -void QGstStructure::free() +QGstStructureView::QGstStructureView(const QUniqueGstStructureHandle &handle) + : QGstStructureView{ handle.get() } { - if (structure) - gst_structure_free(const_cast<GstStructure *>(structure)); - structure = nullptr; } -bool QGstStructure::isNull() const +QUniqueGstStructureHandle QGstStructureView::clone() const +{ + return QUniqueGstStructureHandle{ gst_structure_copy(structure) }; +} + +bool QGstStructureView::isNull() const { return !structure; } -QByteArrayView QGstStructure::name() const +QByteArrayView QGstStructureView::name() const { return gst_structure_get_name(structure); } -QGValue QGstStructure::operator[](const char *name) const +QGValue QGstStructureView::operator[](const char *fieldname) const +{ + return QGValue{ gst_structure_get_value(structure, fieldname) }; +} + +QGstCaps QGstStructureView::caps() const { - return QGValue{ gst_structure_get_value(structure, name) }; + return operator[]("caps").toCaps(); } -QGstStructure QGstStructure::copy() const +QGstTagListHandle QGstStructureView::tags() const { - return gst_structure_copy(structure); + QGValue tags = operator[]("tags"); + if (tags.isNull()) + return {}; + + QGstTagListHandle tagList; + gst_structure_get(structure, "tags", GST_TYPE_TAG_LIST, &tagList, nullptr); + return tagList; } -QSize QGstStructure::resolution() const +QSize QGstStructureView::resolution() const { QSize size; @@ -201,7 +215,7 @@ QSize QGstStructure::resolution() const return size; } -QVideoFrameFormat::PixelFormat QGstStructure::pixelFormat() const +QVideoFrameFormat::PixelFormat QGstStructureView::pixelFormat() const { QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid; @@ -224,7 +238,7 @@ QVideoFrameFormat::PixelFormat QGstStructure::pixelFormat() const return pixelFormat; } -QGRange<float> QGstStructure::frameRateRange() const +QGRange<float> QGstStructureView::frameRateRange() const { float minRate = 0.; float maxRate = 0.; @@ -276,14 +290,14 @@ QGRange<float> QGstStructure::frameRateRange() const return { minRate, maxRate }; } -QGstreamerMessage QGstStructure::getMessage() +QGstreamerMessage QGstStructureView::getMessage() { GstMessage *message = nullptr; gst_structure_get(structure, "message", GST_TYPE_MESSAGE, &message, nullptr); return QGstreamerMessage(message, QGstreamerMessage::HasRef); } -std::optional<Fraction> QGstStructure::pixelAspectRatio() const +std::optional<Fraction> QGstStructureView::pixelAspectRatio() const { gint numerator; gint denominator; @@ -297,7 +311,20 @@ std::optional<Fraction> QGstStructure::pixelAspectRatio() const return std::nullopt; } -QSize QGstStructure::nativeSize() const +// QTBUG-125249: gstreamer tries "to keep the input height (because of interlacing)". Can we align +// the behavior between gstreamer and ffmpeg? +static QSize qCalculateFrameSizeGStreamer(QSize resolution, Fraction par) +{ + if (par.numerator == par.denominator || par.numerator < 1 || par.denominator < 1) + return resolution; + + return QSize{ + resolution.width() * par.numerator / par.denominator, + resolution.height(), + }; +} + +QSize QGstStructureView::nativeSize() const { QSize size = resolution(); if (!size.isValid()) { @@ -307,7 +334,7 @@ QSize QGstStructure::nativeSize() const std::optional<Fraction> par = pixelAspectRatio(); if (par) - size = qCalculateFrameSize(size, *par); + size = qCalculateFrameSizeGStreamer(size, *par); return size; } @@ -329,7 +356,7 @@ std::optional<std::pair<QVideoFrameFormat, GstVideoInfo>> QGstCaps::formatAndVid qt_videoFormatLookup[index].pixelFormat); if (vidInfo.fps_d > 0) - format.setFrameRate(qreal(vidInfo.fps_n) / vidInfo.fps_d); + format.setStreamFrameRate(qreal(vidInfo.fps_n) / vidInfo.fps_d); QVideoFrameFormat::ColorRange range = QVideoFrameFormat::ColorRange_Unknown; switch (vidInfo.colorimetry.range) { @@ -482,6 +509,14 @@ QGstCaps QGstCaps::fromCameraFormat(const QCameraFormat &format) return caps; } +QGstCaps QGstCaps::copy() const +{ + return QGstCaps{ + gst_caps_copy(caps()), + QGstCaps::HasRef, + }; +} + QGstCaps::MemoryFormat QGstCaps::memoryFormat() const { auto *features = gst_caps_get_features(get(), 0); @@ -497,9 +532,11 @@ int QGstCaps::size() const return int(gst_caps_get_size(get())); } -QGstStructure QGstCaps::at(int index) const +QGstStructureView QGstCaps::at(int index) const { - return gst_caps_get_structure(get(), index); + return QGstStructureView{ + gst_caps_get_structure(get(), index), + }; } GstCaps *QGstCaps::caps() const @@ -566,11 +603,11 @@ QGString QGstObject::getString(const char *property) const return QGString(s); } -QGstStructure QGstObject::getStructure(const char *property) const +QGstStructureView QGstObject::getStructure(const char *property) const { GstStructure *s = nullptr; g_object_get(get(), property, &s, nullptr); - return QGstStructure(s); + return QGstStructureView(s); } bool QGstObject::getBool(const char *property) const @@ -648,14 +685,23 @@ GType QGstObject::type() const return G_OBJECT_TYPE(get()); } +QLatin1StringView QGstObject::typeName() const +{ + return QLatin1StringView{ + g_type_name(type()), + }; +} + GstObject *QGstObject::object() const { return get(); } -const char *QGstObject::name() const +QLatin1StringView QGstObject::name() const { - return get() ? GST_OBJECT_NAME(get()) : "(null)"; + using namespace Qt::StringLiterals; + + return get() ? QLatin1StringView{ GST_OBJECT_NAME(get()) } : "(null)"_L1; } // QGObjectHandlerConnection @@ -723,6 +769,28 @@ QGstCaps QGstPad::queryCaps() const return QGstCaps(gst_pad_query_caps(pad(), nullptr), QGstCaps::HasRef); } +QGstTagListHandle QGstPad::tags() const +{ + QGstTagListHandle tagList; + g_object_get(object(), "tags", &tagList, nullptr); + return tagList; +} + +std::optional<QPlatformMediaPlayer::TrackType> QGstPad::inferTrackTypeFromName() const +{ + using namespace Qt::Literals; + QLatin1StringView padName = name(); + + if (padName.startsWith("video_"_L1)) + return QPlatformMediaPlayer::TrackType::VideoStream; + if (padName.startsWith("audio_"_L1)) + return QPlatformMediaPlayer::TrackType::AudioStream; + if (padName.startsWith("text_"_L1)) + return QPlatformMediaPlayer::TrackType::SubtitleStream; + + return std::nullopt; +} + bool QGstPad::isLinked() const { return gst_pad_is_linked(pad()); @@ -850,6 +918,38 @@ QGstElement QGstElement::createFromDevice(GstDevice *device, const char *name) }; } +QGstElement QGstElement::createFromPipelineDescription(const char *str) +{ + QUniqueGErrorHandle error; + QGstElement element{ + gst_parse_launch(str, &error), + QGstElement::NeedsRef, + }; + + if (error) // error does not mean that the element could not be constructed + qWarning() << "gst_parse_launch error:" << error; + + return element; +} + +QGstElement QGstElement::createFromPipelineDescription(const QByteArray &str) +{ + return createFromPipelineDescription(str.constData()); +} + +QGstElementFactoryHandle QGstElement::findFactory(const char *name) +{ + return QGstElementFactoryHandle{ + gst_element_factory_find(name), + QGstElementFactoryHandle::HasRef, + }; +} + +QGstElementFactoryHandle QGstElement::findFactory(const QByteArray &name) +{ + return findFactory(name.constData()); +} + QGstPad QGstElement::staticPad(const char *name) const { return QGstPad(gst_element_get_static_pad(element(), name), HasRef); @@ -901,14 +1001,23 @@ GstStateChangeReturn QGstElement::setState(GstState state) bool QGstElement::setStateSync(GstState state, std::chrono::nanoseconds timeout) { + if (state == GST_STATE_NULL) { + // QTBUG-125251: when changing pipeline state too quickly between NULL->PAUSED->NULL there + // may be a pending task to activate pads while we try to switch to NULL. This can cause an + // assertion failure in gstreamer. we therefore finish the state change when called on a bin + // or pipeline. + if (qIsGstObjectOfType<GstBin>(element())) + finishStateChange(); + } + GstStateChangeReturn change = gst_element_set_state(element(), state); - if (change == GST_STATE_CHANGE_ASYNC) { + if (change == GST_STATE_CHANGE_ASYNC) change = gst_element_get_state(element(), nullptr, &state, timeout.count()); - } -#ifndef QT_NO_DEBUG - if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) + + if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) { qWarning() << "Could not change state of" << name() << "to" << state << change; -#endif + dumpPipelineGraph("setStatSyncFailure"); + } return change == GST_STATE_CHANGE_SUCCESS; } @@ -924,10 +1033,10 @@ bool QGstElement::finishStateChange(std::chrono::nanoseconds timeout) GstStateChangeReturn change = gst_element_get_state(element(), &state, &pending, timeout.count()); -#ifndef QT_NO_DEBUG - if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) + if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) { qWarning() << "Could not finish change state of" << name() << change << state << pending; -#endif + dumpPipelineGraph("finishStateChangeFailure"); + } return change == GST_STATE_CHANGE_SUCCESS; } @@ -951,6 +1060,48 @@ void QGstElement::sendEos() const sendEvent(gst_event_new_eos()); } +std::optional<std::chrono::nanoseconds> QGstElement::duration() const +{ + gint64 d; + if (!gst_element_query_duration(element(), GST_FORMAT_TIME, &d)) { + qDebug() << "QGstElement: failed to query duration"; + return std::nullopt; + } + return std::chrono::nanoseconds{ d }; +} + +std::optional<std::chrono::milliseconds> QGstElement::durationInMs() const +{ + using namespace std::chrono; + auto dur = duration(); + if (dur) + return round<milliseconds>(*dur); + return std::nullopt; +} + +std::optional<std::chrono::nanoseconds> QGstElement::position() const +{ + QGstQueryHandle &query = positionQuery(); + + gint64 pos; + if (gst_element_query(element(), query.get())) { + gst_query_parse_position(query.get(), nullptr, &pos); + return std::chrono::nanoseconds{ pos }; + } + + qDebug() << "QGstElement: failed to query position"; + return std::nullopt; +} + +std::optional<std::chrono::milliseconds> QGstElement::positionInMs() const +{ + using namespace std::chrono; + auto pos = position(); + if (pos) + return round<milliseconds>(*pos); + return std::nullopt; +} + GstClockTime QGstElement::baseTime() const { return gst_element_get_base_time(element()); @@ -991,6 +1142,27 @@ QGstPipeline QGstElement::getPipeline() const } } +void QGstElement::dumpPipelineGraph(const char *filename) const +{ + static const bool dumpEnabled = qEnvironmentVariableIsSet("GST_DEBUG_DUMP_DOT_DIR"); + if (dumpEnabled) { + QGstPipeline pipeline = getPipeline(); + if (pipeline) + pipeline.dumpGraph(filename); + } +} + +QGstQueryHandle &QGstElement::positionQuery() const +{ + if (Q_UNLIKELY(!m_positionQuery)) + m_positionQuery = QGstQueryHandle{ + gst_query_new_position(GST_FORMAT_TIME), + QGstQueryHandle::HasRef, + }; + + return m_positionQuery; +} + // QGstBin QGstBin QGstBin::create(const char *name) @@ -1008,6 +1180,36 @@ QGstBin QGstBin::createFromFactory(const char *factory, const char *name) }; } +QGstBin QGstBin::createFromPipelineDescription(const QByteArray &pipelineDescription, + const char *name, bool ghostUnlinkedPads) +{ + return createFromPipelineDescription(pipelineDescription.constData(), name, ghostUnlinkedPads); +} + +QGstBin QGstBin::createFromPipelineDescription(const char *pipelineDescription, const char *name, + bool ghostUnlinkedPads) +{ + QUniqueGErrorHandle error; + + GstElement *element = + gst_parse_bin_from_description_full(pipelineDescription, ghostUnlinkedPads, + /*context=*/nullptr, GST_PARSE_FLAG_NONE, &error); + + if (!element) { + qWarning() << "Failed to make element from pipeline description" << pipelineDescription + << error; + return QGstBin{}; + } + + if (name) + gst_element_set_name(element, name); + + return QGstBin{ + element, + NeedsRef, + }; +} + QGstBin::QGstBin(GstBin *bin, RefMode mode) : QGstElement{ qGstCheckedCast<GstElement>(bin), @@ -1041,12 +1243,15 @@ void QGstBin::dumpGraph(const char *fileNamePrefix) if (isNull()) return; - GST_DEBUG_BIN_TO_DOT_FILE(bin(), - GstDebugGraphDetails(GST_DEBUG_GRAPH_SHOW_ALL - | GST_DEBUG_GRAPH_SHOW_MEDIA_TYPE - | GST_DEBUG_GRAPH_SHOW_NON_DEFAULT_PARAMS - | GST_DEBUG_GRAPH_SHOW_STATES), - fileNamePrefix); + GST_DEBUG_BIN_TO_DOT_FILE(bin(), GST_DEBUG_GRAPH_SHOW_VERBOSE, fileNamePrefix); +} + +QGstElement QGstBin::findByName(const char *name) +{ + return QGstElement{ + gst_bin_get_by_name(bin(), name), + QGstElement::NeedsRef, + }; } // QGstBaseSink @@ -1059,6 +1264,11 @@ QGstBaseSink::QGstBaseSink(GstBaseSink *element, RefMode mode) { } +void QGstBaseSink::setSync(bool arg) +{ + gst_base_sink_set_sync(baseSink(), arg ? TRUE : FALSE); +} + GstBaseSink *QGstBaseSink::baseSink() const { return qGstCheckedCast<GstBaseSink>(element()); @@ -1105,6 +1315,18 @@ GstAppSink *QGstAppSink::appSink() const return qGstCheckedCast<GstAppSink>(element()); } +# if GST_CHECK_VERSION(1, 24, 0) +void QGstAppSink::setMaxBufferTime(std::chrono::nanoseconds ns) +{ + gst_app_sink_set_max_time(appSink(), qGstClockTimeFromChrono(ns)); +} +# endif + +void QGstAppSink::setMaxBuffers(int n) +{ + gst_app_sink_set_max_buffers(appSink(), n); +} + void QGstAppSink::setCaps(const QGstCaps &caps) { gst_app_sink_set_caps(appSink(), caps.caps()); @@ -1161,4 +1383,10 @@ GstFlowReturn QGstAppSrc::pushBuffer(GstBuffer *buffer) #endif +QString qGstErrorMessageCannotFindElement(std::string_view element) +{ + return QStringLiteral("Could not find the %1 GStreamer element") + .arg(QLatin1StringView(element)); +} + QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp b/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp index ee28a5c45..413b02f44 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp @@ -4,6 +4,8 @@ #include "qgst_debug_p.h" #include "qgstreamermessage_p.h" +#include <gst/gstclock.h> + QT_BEGIN_NAMESPACE // NOLINTBEGIN(performance-unnecessary-value-param) @@ -18,7 +20,7 @@ QDebug operator<<(QDebug dbg, const QGstCaps &caps) return dbg << caps.caps(); } -QDebug operator<<(QDebug dbg, const QGstStructure &structure) +QDebug operator<<(QDebug dbg, const QGstStructureView &structure) { return dbg << structure.structure; } @@ -43,6 +45,21 @@ QDebug operator<<(QDebug dbg, const QUniqueGStringHandle &handle) return dbg << handle.get(); } +QDebug operator<<(QDebug dbg, const QGstStreamCollectionHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstStreamHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstTagListHandle &handle) +{ + return dbg << handle.get(); +} + QDebug operator<<(QDebug dbg, const QGstElement &element) { return dbg << element.element(); @@ -155,27 +172,126 @@ QDebug operator<<(QDebug dbg, const GstDevice *device) return dbg; } +namespace { + +struct Timepoint +{ + explicit Timepoint(guint64 us) : ts{ us } { } + guint64 ts; +}; + +QDebug operator<<(QDebug dbg, Timepoint ts) +{ + char buffer[128]; + snprintf(buffer, sizeof(buffer), "%" GST_TIME_FORMAT, GST_TIME_ARGS(ts.ts)); + dbg << buffer; + return dbg; +} + +} // namespace + QDebug operator<<(QDebug dbg, const GstMessage *msg) { QDebugStateSaver saver(dbg); dbg.nospace(); + dbg << GST_MESSAGE_TYPE_NAME(msg) << ", Source: " << GST_MESSAGE_SRC_NAME(msg); + if (GST_MESSAGE_TIMESTAMP(msg) != 0xFFFFFFFFFFFFFFFF) + dbg << ", Timestamp: " << GST_MESSAGE_TIMESTAMP(msg); + switch (msg->type) { case GST_MESSAGE_ERROR: { QUniqueGErrorHandle err; QGString debug; gst_message_parse_error(const_cast<GstMessage *>(msg), &err, &debug); - dbg << GST_MESSAGE_TYPE_NAME(msg) << ", Source: " << GST_MESSAGE_SRC_NAME(msg) - << ", Timestamp: " << GST_MESSAGE_TIMESTAMP(msg) << ", Error: " << err << " (" << debug - << ")"; + dbg << ", Error: " << err << " (" << debug << ")"; break; } - default: { - dbg << GST_MESSAGE_TYPE_NAME(msg) << ", Source: " << GST_MESSAGE_SRC_NAME(msg) - << ", Timestamp: " << GST_MESSAGE_TIMESTAMP(msg); + case GST_MESSAGE_WARNING: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_warning(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Warning: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_INFO: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_info(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Info: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_TAG: { + QGstTagListHandle tagList; + gst_message_parse_tag(const_cast<GstMessage *>(msg), &tagList); + + dbg << ", Tags: " << tagList; + break; + } + + case GST_MESSAGE_QOS: { + gboolean live; + guint64 running_time; + guint64 stream_time; + guint64 timestamp; + guint64 duration; + + gst_message_parse_qos(const_cast<GstMessage *>(msg), &live, &running_time, &stream_time, + ×tamp, &duration); + + dbg << ", Live: " << bool(live) << ", Running time: " << Timepoint{ running_time } + << ", Stream time: " << Timepoint{ stream_time } + << ", Timestamp: " << Timepoint{ timestamp } << ", Duration: " << Timepoint{ duration }; + break; + } + + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(const_cast<GstMessage *>(msg), &oldState, &newState, + &pending); + + dbg << ", Transition: " << oldState << "->" << newState; + + if (pending != GST_STATE_VOID_PENDING) + dbg << ", Pending State: " << pending; + break; } + + case GST_MESSAGE_STREAM_COLLECTION: { + QGstStreamCollectionHandle collection; + gst_message_parse_stream_collection(const_cast<GstMessage *>(msg), &collection); + + dbg << ", " << collection; + break; + } + + case GST_MESSAGE_STREAMS_SELECTED: { + QGstStreamCollectionHandle collection; + gst_message_parse_streams_selected(const_cast<GstMessage *>(msg), &collection); + + dbg << ", " << collection; + break; + } + + case GST_MESSAGE_STREAM_STATUS: { + GstStreamStatusType streamStatus; + gst_message_parse_stream_status(const_cast<GstMessage *>(msg), &streamStatus, nullptr); + + dbg << ", Stream Status: " << streamStatus; + break; + } + + default: + break; } return dbg; } @@ -208,6 +324,50 @@ QDebug operator<<(QDebug dbg, const GstPadTemplate *padTemplate) return dbg; } +QDebug operator<<(QDebug dbg, const GstStreamCollection *streamCollection) +{ + GstStreamCollection *collection = const_cast<GstStreamCollection *>(streamCollection); + guint size = gst_stream_collection_get_size(collection); + + dbg << "Stream Collection: {"; + for (guint index = 0; index != size; ++index) { + dbg << gst_stream_collection_get_stream(collection, index); + if (index + 1 != size) + dbg << ", "; + } + + dbg << "}"; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstStream *cstream) +{ + GstStream *stream = const_cast<GstStream *>(cstream); + + dbg << "GstStream { "; + dbg << "Type: " << gst_stream_type_get_name(gst_stream_get_stream_type(stream)); + + QGstTagListHandle tagList{ + gst_stream_get_tags(stream), + QGstTagListHandle::HasRef, + }; + + if (tagList) + dbg << ", Tags: " << tagList; + + QGstCaps caps{ + gst_stream_get_caps(stream), + QGstCaps::HasRef, + }; + + if (caps) + dbg << ", Caps: " << caps; + + dbg << "}"; + + return dbg; +} + QDebug operator<<(QDebug dbg, GstState state) { return dbg << gst_element_state_get_name(state); @@ -228,19 +388,40 @@ QDebug operator<<(QDebug dbg, GstMessageType type) return dbg << gst_message_type_get_name(type); } +#define ADD_ENUM_SWITCH(value) \ + case value: \ + return dbg << #value; \ + static_assert(true, "enforce semicolon") + QDebug operator<<(QDebug dbg, GstPadDirection direction) { switch (direction) { - case GST_PAD_UNKNOWN: - return dbg << "GST_PAD_UNKNOWN"; - case GST_PAD_SRC: - return dbg << "GST_PAD_SRC"; - case GST_PAD_SINK: - return dbg << "GST_PAD_SINK"; + ADD_ENUM_SWITCH(GST_PAD_UNKNOWN); + ADD_ENUM_SWITCH(GST_PAD_SRC); + ADD_ENUM_SWITCH(GST_PAD_SINK); + default: + Q_UNREACHABLE_RETURN(dbg); + } +} + +QDebug operator<<(QDebug dbg, GstStreamStatusType type) +{ + switch (type) { + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_CREATE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_ENTER); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_LEAVE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_DESTROY); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_START); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_PAUSE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_STOP); + default: + Q_UNREACHABLE_RETURN(dbg); } return dbg; } +#undef ADD_ENUM_SWITCH + QDebug operator<<(QDebug dbg, const GValue *value) { switch (G_VALUE_TYPE(value)) { @@ -316,4 +497,69 @@ QDebug operator<<(QDebug dbg, const GError *error) return dbg << error->message; } +QCompactGstMessageAdaptor::QCompactGstMessageAdaptor(const QGstreamerMessage &m) + : QCompactGstMessageAdaptor{ + m.message(), + } +{ +} + +QCompactGstMessageAdaptor::QCompactGstMessageAdaptor(GstMessage *m) + : msg{ + m, + } +{ +} + +QDebug operator<<(QDebug dbg, const QCompactGstMessageAdaptor &m) +{ + std::optional<QDebugStateSaver> saver(dbg); + dbg.nospace(); + + switch (GST_MESSAGE_TYPE(m.msg)) { + case GST_MESSAGE_ERROR: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(m.msg, &err, &debug); + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_WARNING: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_warning(m.msg, &err, &debug); + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_INFO: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_info(m.msg, &err, &debug); + + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(m.msg, &oldState, &newState, &pending); + + dbg << oldState << " -> " << newState; + if (pending != GST_STATE_VOID_PENDING) + dbg << " (pending: " << pending << ")"; + return dbg; + } + + default: { + saver.reset(); + return dbg << m.msg; + } + } +} + QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h b/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h index d64c240c6..df13c6c13 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h @@ -23,7 +23,7 @@ QT_BEGIN_NAMESPACE class QGstreamerMessage; QDebug operator<<(QDebug, const QGstCaps &); -QDebug operator<<(QDebug, const QGstStructure &); +QDebug operator<<(QDebug, const QGstStructureView &); QDebug operator<<(QDebug, const QGstElement &); QDebug operator<<(QDebug, const QGstPad &); QDebug operator<<(QDebug, const QGString &); @@ -31,6 +31,9 @@ QDebug operator<<(QDebug, const QGValue &); QDebug operator<<(QDebug, const QGstreamerMessage &); QDebug operator<<(QDebug, const QUniqueGErrorHandle &); QDebug operator<<(QDebug, const QUniqueGStringHandle &); +QDebug operator<<(QDebug, const QGstStreamCollectionHandle &); +QDebug operator<<(QDebug, const QGstStreamHandle &); +QDebug operator<<(QDebug, const QGstTagListHandle &); QDebug operator<<(QDebug, const GstCaps *); QDebug operator<<(QDebug, const GstVideoInfo *); @@ -44,16 +47,28 @@ QDebug operator<<(QDebug, const GstTagList *); QDebug operator<<(QDebug, const GstQuery *); QDebug operator<<(QDebug, const GstEvent *); QDebug operator<<(QDebug, const GstPadTemplate *); +QDebug operator<<(QDebug, const GstStreamCollection *); +QDebug operator<<(QDebug, const GstStream *); QDebug operator<<(QDebug, GstState); QDebug operator<<(QDebug, GstStateChange); QDebug operator<<(QDebug, GstStateChangeReturn); QDebug operator<<(QDebug, GstMessageType); QDebug operator<<(QDebug, GstPadDirection); +QDebug operator<<(QDebug, GstStreamStatusType); QDebug operator<<(QDebug, const GValue *); QDebug operator<<(QDebug, const GError *); +struct QCompactGstMessageAdaptor +{ + explicit QCompactGstMessageAdaptor(const QGstreamerMessage &m); + explicit QCompactGstMessageAdaptor(GstMessage *m); + GstMessage *msg; +}; + +QDebug operator<<(QDebug, const QCompactGstMessageAdaptor &); + QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h b/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h index b72e92db1..e813f4181 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h @@ -46,7 +46,7 @@ struct QSharedHandle : private QUniqueHandle<HandleTraits> } QSharedHandle(const QSharedHandle &o) - : QSharedHandle{ + : BaseClass{ HandleTraits::ref(o.get()), } { @@ -163,6 +163,18 @@ struct QUniqueGErrorHandleTraits } }; + +struct QUniqueGstDateTimeHandleTraits +{ + using Type = GstDateTime *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_date_time_unref(handle); + return true; + } +}; + struct QFileDescriptorHandleTraits { using Type = int; @@ -213,7 +225,8 @@ struct QGstMiniObjectHandleHelper static Type ref(Type handle) noexcept { - gst_mini_object_ref(GST_MINI_OBJECT_CAST(handle)); + if (GST_MINI_OBJECT_CAST(handle)) + gst_mini_object_ref(GST_MINI_OBJECT_CAST(handle)); return handle; } }; @@ -226,10 +239,12 @@ struct QGstMiniObjectHandleHelper using QGstClockHandle = QGstImpl::QGstHandleHelper<GstClock>::UniqueHandle; using QGstElementHandle = QGstImpl::QGstHandleHelper<GstElement>::UniqueHandle; -using QGstElementFactoryHandle = QGstImpl::QGstHandleHelper<GstElementFactory>::UniqueHandle; +using QGstElementFactoryHandle = QGstImpl::QGstHandleHelper<GstElementFactory>::SharedHandle; using QGstDeviceHandle = QGstImpl::QGstHandleHelper<GstDevice>::SharedHandle; using QGstDeviceMonitorHandle = QGstImpl::QGstHandleHelper<GstDeviceMonitor>::UniqueHandle; using QGstBusHandle = QGstImpl::QGstHandleHelper<GstBus>::UniqueHandle; +using QGstStreamCollectionHandle = QGstImpl::QGstHandleHelper<GstStreamCollection>::SharedHandle; +using QGstStreamHandle = QGstImpl::QGstHandleHelper<GstStream>::SharedHandle; using QGstTagListHandle = QGstImpl::QSharedHandle<QGstImpl::QGstTagListHandleTraits>; using QGstSampleHandle = QGstImpl::QSharedHandle<QGstImpl::QGstSampleHandleTraits>; @@ -237,10 +252,13 @@ using QGstSampleHandle = QGstImpl::QSharedHandle<QGstImpl::QGstSampleHandleTrait using QUniqueGstStructureHandle = QUniqueHandle<QGstImpl::QUniqueGstStructureHandleTraits>; using QUniqueGStringHandle = QUniqueHandle<QGstImpl::QUniqueGStringHandleTraits>; using QUniqueGErrorHandle = QUniqueHandle<QGstImpl::QUniqueGErrorHandleTraits>; +using QUniqueGstDateTimeHandle = QUniqueHandle<QGstImpl::QUniqueGstDateTimeHandleTraits>; using QFileDescriptorHandle = QUniqueHandle<QGstImpl::QFileDescriptorHandleTraits>; +using QGstBufferHandle = QGstImpl::QGstMiniObjectHandleHelper<GstBuffer>::SharedHandle; using QGstContextHandle = QGstImpl::QGstMiniObjectHandleHelper<GstContext>::UniqueHandle; using QGstGstDateTimeHandle = QGstImpl::QGstMiniObjectHandleHelper<GstDateTime>::SharedHandle; using QGstPluginFeatureHandle = QGstImpl::QGstHandleHelper<GstPluginFeature>::SharedHandle; +using QGstQueryHandle = QGstImpl::QGstMiniObjectHandleHelper<GstQuery>::SharedHandle; #if QT_CONFIG(gstreamer_gl) using QGstGLContextHandle = QGstImpl::QGstHandleHelper<GstGLContext>::UniqueHandle; diff --git a/src/plugins/multimedia/gstreamer/common/qgst_p.h b/src/plugins/multimedia/gstreamer/common/qgst_p.h index ab264f552..865b5895d 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgst_p.h @@ -15,21 +15,21 @@ // We mean it. // -#include <common/qgst_handle_types_p.h> - -#include <private/qtmultimediaglobal_p.h> -#include <private/qmultimediautils_p.h> - #include <QtCore/qdebug.h> #include <QtCore/qlist.h> #include <QtCore/qsemaphore.h> #include <QtMultimedia/qaudioformat.h> #include <QtMultimedia/qvideoframe.h> +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtMultimedia/private/qmultimediautils_p.h> +#include <QtMultimedia/private/qplatformmediaplayer_p.h> #include <gst/gst.h> #include <gst/video/video-info.h> +#include "qgst_handle_types_p.h" + #include <type_traits> #if QT_CONFIG(gstreamer_photography) @@ -80,6 +80,29 @@ struct GstObjectTraits }; \ static_assert(true, "ensure semicolon") +#define QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE(ClassName, MACRO_LABEL) \ + template <> \ + struct GstObjectTraits<ClassName> \ + { \ + using Type = ClassName; \ + template <typename U> \ + static bool isObjectOfType(U *arg) \ + { \ + return GST_IS_##MACRO_LABEL(arg); \ + } \ + template <typename U> \ + static Type *cast(U *arg) \ + { \ + return checked_cast(arg); \ + } \ + template <typename U> \ + static Type *checked_cast(U *arg) \ + { \ + return GST_##MACRO_LABEL(arg); \ + } \ + }; \ + static_assert(true, "ensure semicolon") + QGST_DEFINE_CAST_TRAITS(GstBin, BIN); QGST_DEFINE_CAST_TRAITS(GstClock, CLOCK); QGST_DEFINE_CAST_TRAITS(GstElement, ELEMENT); @@ -89,6 +112,8 @@ QGST_DEFINE_CAST_TRAITS(GstPipeline, PIPELINE); QGST_DEFINE_CAST_TRAITS(GstBaseSink, BASE_SINK); QGST_DEFINE_CAST_TRAITS(GstBaseSrc, BASE_SRC); +QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE(GstTagSetter, TAG_SETTER); + #if QT_CONFIG(gstreamer_app) QGST_DEFINE_CAST_TRAITS(GstAppSink, APP_SINK); QGST_DEFINE_CAST_TRAITS(GstAppSrc, APP_SRC); @@ -116,10 +141,18 @@ struct GstObjectTraits<GObject> }; #undef QGST_DEFINE_CAST_TRAITS +#undef QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE } // namespace QGstImpl template <typename DestinationType, typename SourceType> +bool qIsGstObjectOfType(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + return arg && Traits::isObjectOfType(arg); +} + +template <typename DestinationType, typename SourceType> DestinationType *qGstSafeCast(SourceType *arg) { using Traits = QGstImpl::GstObjectTraits<DestinationType>; @@ -138,7 +171,7 @@ DestinationType *qGstCheckedCast(SourceType *arg) } class QSize; -class QGstStructure; +class QGstStructureView; class QGstCaps; class QGstPipelinePrivate; class QCameraFormat; @@ -179,7 +212,7 @@ public: std::optional<QGRange<float>> getFractionRange() const; std::optional<QGRange<int>> toIntRange() const; - QGstStructure toStructure() const; + QGstStructureView toStructure() const; QGstCaps toCaps() const; bool isList() const; @@ -277,18 +310,21 @@ protected: class QGstreamerMessage; -class QGstStructure +class QGstStructureView { public: const GstStructure *structure = nullptr; - QGstStructure() = default; - QGstStructure(const GstStructure *s); - void free(); + explicit QGstStructureView(const GstStructure *); + explicit QGstStructureView(const QUniqueGstStructureHandle &); - bool isNull() const; + QUniqueGstStructureHandle clone() const; + bool isNull() const; QByteArrayView name() const; - QGValue operator[](const char *name) const; + QGValue operator[](const char *fieldname) const; + + QGstCaps caps() const; + QGstTagListHandle tags() const; QSize resolution() const; QVideoFrameFormat::PixelFormat pixelFormat() const; @@ -296,8 +332,6 @@ public: QGstreamerMessage getMessage(); std::optional<Fraction> pixelAspectRatio() const; QSize nativeSize() const; - - QGstStructure copy() const; }; template <> @@ -321,7 +355,7 @@ public: enum MemoryFormat { CpuMemory, GLTexture, DMABuf }; int size() const; - QGstStructure at(int index) const; + QGstStructureView at(int index) const; GstCaps *caps() const; MemoryFormat memoryFormat() const; @@ -333,6 +367,8 @@ public: static QGstCaps create(); static QGstCaps fromCameraFormat(const QCameraFormat &format); + + QGstCaps copy() const; }; template <> @@ -367,7 +403,7 @@ public: void set(const char *property, const QGstCaps &c); QGString getString(const char *property) const; - QGstStructure getStructure(const char *property) const; + QGstStructureView getStructure(const char *property) const; bool getBool(const char *property) const; uint getUInt(const char *property) const; int getInt(const char *property) const; @@ -381,8 +417,9 @@ public: void disconnect(gulong handlerId); GType type() const; + QLatin1StringView typeName() const; GstObject *object() const; - const char *name() const; + QLatin1StringView name() const; }; class QGObjectHandlerConnection @@ -443,6 +480,11 @@ public: QGstCaps currentCaps() const; QGstCaps queryCaps() const; + QGstTagListHandle tags() const; + + std::optional<QPlatformMediaPlayer::TrackType> + inferTrackTypeFromName() const; // for decodebin3 etc + bool isLinked() const; bool link(const QGstPad &sink) const; bool unlink(const QGstPad &sink) const; @@ -531,6 +573,11 @@ public: const char *name = nullptr); static QGstElement createFromDevice(const QGstDeviceHandle &, const char *name = nullptr); static QGstElement createFromDevice(GstDevice *, const char *name = nullptr); + static QGstElement createFromPipelineDescription(const char *); + static QGstElement createFromPipelineDescription(const QByteArray &); + + static QGstElementFactoryHandle findFactory(const char *); + static QGstElementFactoryHandle findFactory(const QByteArray &name); QGstPad staticPad(const char *name) const; QGstPad src() const; @@ -550,6 +597,11 @@ public: void sendEvent(GstEvent *event) const; void sendEos() const; + std::optional<std::chrono::nanoseconds> duration() const; + std::optional<std::chrono::milliseconds> durationInMs() const; + std::optional<std::chrono::nanoseconds> position() const; + std::optional<std::chrono::milliseconds> positionInMs() const; + template <auto Member, typename T> QGObjectHandlerConnection onPadAdded(T *instance) { @@ -599,6 +651,11 @@ public: QGstElement getParent() const; QGstPipeline getPipeline() const; + void dumpPipelineGraph(const char *filename) const; + +private: + QGstQueryHandle &positionQuery() const; + mutable QGstQueryHandle m_positionQuery; }; template <typename... Ts> @@ -642,6 +699,12 @@ public: explicit QGstBin(GstBin *bin, RefMode mode = NeedsRef); static QGstBin create(const char *name); static QGstBin createFromFactory(const char *factory, const char *name); + static QGstBin createFromPipelineDescription(const QByteArray &pipelineDescription, + const char *name = nullptr, + bool ghostUnlinkedPads = false); + static QGstBin createFromPipelineDescription(const char *pipelineDescription, + const char *name = nullptr, + bool ghostUnlinkedPads = false); template <typename... Ts> std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> add(const Ts &...ts) @@ -678,6 +741,8 @@ public: bool syncChildrenState(); void dumpGraph(const char *fileNamePrefix); + + QGstElement findByName(const char *); }; class QGstBaseSink : public QGstElement @@ -692,6 +757,8 @@ public: QGstBaseSink &operator=(const QGstBaseSink &) = default; QGstBaseSink &operator=(QGstBaseSink &&) noexcept = default; + void setSync(bool); + GstBaseSink *baseSink() const; }; @@ -727,6 +794,11 @@ public: GstAppSink *appSink() const; + void setMaxBuffers(int); +# if GST_CHECK_VERSION(1, 24, 0) + void setMaxBufferTime(std::chrono::nanoseconds); +# endif + void setCaps(const QGstCaps &caps); void setCallbacks(GstAppSinkCallbacks &callbacks, gpointer user_data, GDestroyNotify notify); @@ -756,10 +828,24 @@ public: #endif -inline QString errorMessageCannotFindElement(std::string_view element) +inline GstClockTime qGstClockTimeFromChrono(std::chrono::nanoseconds ns) { - return QStringLiteral("Could not find the %1 GStreamer element") - .arg(QLatin1StringView(element)); + return ns.count(); +} + +QString qGstErrorMessageCannotFindElement(std::string_view element); + +template <typename Arg, typename... Args> +std::optional<QString> qGstErrorMessageIfElementsNotAvailable(const Arg &arg, Args... args) +{ + QGstElementFactoryHandle factory = QGstElement::findFactory(arg); + if (!factory) + return qGstErrorMessageCannotFindElement(arg); + + if constexpr (sizeof...(args) != 0) + return qGstErrorMessageIfElementsNotAvailable(args...); + else + return std::nullopt; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp b/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp index 99af8443c..3c345de82 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp @@ -16,7 +16,7 @@ QMaybe<QGstAppSource *> QGstAppSource::create(QObject *parent) { QGstAppSrc appsrc = QGstAppSrc::create("appsrc"); if (!appsrc) - return errorMessageCannotFindElement("appsrc"); + return qGstErrorMessageCannotFindElement("appsrc"); return new QGstAppSource(appsrc, parent); } diff --git a/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp b/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp index 392898245..c92a12764 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp @@ -14,11 +14,16 @@ QT_BEGIN_NAMESPACE +static constexpr GstSeekFlags rateChangeSeekFlags = +#if GST_CHECK_VERSION(1, 18, 0) + GST_SEEK_FLAG_INSTANT_RATE_CHANGE; +#else + GST_SEEK_FLAG_FLUSH; +#endif + class QGstPipelinePrivate : public QObject { - Q_OBJECT public: - int m_ref = 0; guint m_tag = 0; GstBus *m_bus = nullptr; @@ -27,7 +32,7 @@ public: QList<QGstreamerSyncMessageFilter*> syncFilters; QList<QGstreamerBusMessageFilter*> busFilters; bool inStoppedState = true; - mutable qint64 m_position = 0; + mutable std::chrono::nanoseconds m_position{}; double m_rate = 1.; bool m_flushOnConfigChanges = false; bool m_pendingFlush = false; @@ -46,8 +51,21 @@ public: void installMessageFilter(QGstreamerBusMessageFilter *filter); void removeMessageFilter(QGstreamerBusMessageFilter *filter); - static GstBusSyncReply syncGstBusFilter(GstBus* bus, GstMessage* message, QGstPipelinePrivate *d) + void processMessage(const QGstreamerMessage &msg) { + for (QGstreamerBusMessageFilter *filter : std::as_const(busFilters)) { + if (filter->processBusMessage(msg)) + break; + } + } + +private: + static GstBusSyncReply syncGstBusFilter(GstBus *bus, GstMessage *message, + QGstPipelinePrivate *d) + { + if (!message) + return GST_BUS_PASS; + Q_UNUSED(bus); QMutexLocker lock(&d->filterMutex); @@ -62,31 +80,17 @@ public: return GST_BUS_PASS; } -private Q_SLOTS: - void interval() - { - GstMessage* message; - while ((message = gst_bus_poll(m_bus, GST_MESSAGE_ANY, 0)) != nullptr) { - processMessage(message); - gst_message_unref(message); - } - } - void doProcessMessage(const QGstreamerMessage& msg) - { - for (QGstreamerBusMessageFilter *filter : std::as_const(busFilters)) { - if (filter->processBusMessage(msg)) - break; - } - } - -private: void processMessage(GstMessage *message) { + if (!message) + return; + QGstreamerMessage msg{ message, QGstreamerMessage::NeedsRef, }; - doProcessMessage(msg); + + processMessage(msg); } static gboolean busCallback(GstBus *, GstMessage *message, gpointer data) @@ -106,7 +110,13 @@ QGstPipelinePrivate::QGstPipelinePrivate(GstBus* bus, QObject* parent) if (!hasGlib) { m_intervalTimer = new QTimer(this); m_intervalTimer->setInterval(250); - connect(m_intervalTimer, SIGNAL(timeout()), SLOT(interval())); + QObject::connect(m_intervalTimer, &QTimer::timeout, this, [this] { + GstMessage *message; + while ((message = gst_bus_poll(m_bus, GST_MESSAGE_ANY, 0)) != nullptr) { + processMessage(message); + gst_message_unref(message); + } + }); m_intervalTimer->start(); } else { m_tag = gst_bus_add_watch_full(bus, G_PRIORITY_DEFAULT, busCallback, this, nullptr); @@ -235,6 +245,16 @@ GstStateChangeReturn QGstPipeline::setState(GstState state) return retval; } +void QGstPipeline::processMessages(GstMessageType types) +{ + QGstPipelinePrivate *d = getPrivate(); + QGstreamerMessage message{ + gst_bus_pop_filtered(d->m_bus, types), + QGstreamerMessage::HasRef, + }; + d->processMessage(message); +} + void QGstPipeline::dumpGraph(const char *fileName) { if (isNull()) @@ -268,8 +288,8 @@ void QGstPipeline::beginConfig() break; } case GST_STATE_CHANGE_FAILURE: { - // should not happen - qCritical() << "QGstPipeline::beginConfig: state change failure"; + qDebug() << "QGstPipeline::beginConfig: state change failure"; + dumpGraph("beginConfigFailure"); break; } @@ -301,48 +321,45 @@ void QGstPipeline::endConfig() void QGstPipeline::flush() { - QGstPipelinePrivate *d = getPrivate(); - seek(position(), d->m_rate); + seek(position()); } -bool QGstPipeline::seek(qint64 pos, double rate) +void QGstPipeline::seek(std::chrono::nanoseconds pos, double rate) { + using namespace std::chrono_literals; + QGstPipelinePrivate *d = getPrivate(); - // always adjust the rate, so it can be set before playback starts + // always adjust the rate, so it can be set before playback starts // setting position needs a loaded media file that's seekable - d->m_rate = rate; - qint64 from = rate > 0 ? pos : 0; - qint64 to = rate > 0 ? duration() : pos; - bool success = gst_element_seek(element(), rate, GST_FORMAT_TIME, - GstSeekFlags(GST_SEEK_FLAG_FLUSH), - GST_SEEK_TYPE_SET, from, - GST_SEEK_TYPE_SET, to); - if (!success) - return false; + + bool success = (rate > 0) + ? gst_element_seek(element(), d->m_rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, pos.count(), GST_SEEK_TYPE_END, 0) + : gst_element_seek(element(), d->m_rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_SET, pos.count()); + + if (!success) { + qDebug() << "seek: gst_element_seek failed" << pos; + return; + } d->m_position = pos; - return true; } -bool QGstPipeline::setPlaybackRate(double rate) +void QGstPipeline::seek(std::chrono::nanoseconds pos) +{ + seek(pos, getPrivate()->m_rate); +} + +void QGstPipeline::setPlaybackRate(double rate) { QGstPipelinePrivate *d = getPrivate(); if (rate == d->m_rate) - return false; - - constexpr GstSeekFlags seekFlags = -#if GST_CHECK_VERSION(1, 18, 0) - GST_SEEK_FLAG_INSTANT_RATE_CHANGE; -#else - GST_SEEK_FLAG_FLUSH; -#endif + return; - bool success = gst_element_seek(element(), rate, GST_FORMAT_TIME, seekFlags, GST_SEEK_TYPE_NONE, - GST_CLOCK_TIME_NONE, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE); - if (success) - d->m_rate = rate; + d->m_rate = rate; - return success; + applyPlaybackRate(/*instantRateChange =*/true); } double QGstPipeline::playbackRate() const @@ -351,27 +368,39 @@ double QGstPipeline::playbackRate() const return d->m_rate; } -bool QGstPipeline::setPosition(qint64 pos) +void QGstPipeline::applyPlaybackRate(bool instantRateChange) { QGstPipelinePrivate *d = getPrivate(); - return seek(pos, d->m_rate); + + bool success = gst_element_seek(element(), d->m_rate, GST_FORMAT_UNDEFINED, + instantRateChange ? rateChangeSeekFlags : GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE, GST_SEEK_TYPE_NONE, + GST_CLOCK_TIME_NONE); + if (!success) + qDebug() << "setPlaybackRate: gst_element_seek failed"; } -qint64 QGstPipeline::position() const +void QGstPipeline::setPosition(std::chrono::nanoseconds pos) +{ + seek(pos); +} + +std::chrono::nanoseconds QGstPipeline::position() const { - gint64 pos; QGstPipelinePrivate *d = getPrivate(); - if (gst_element_query_position(element(), GST_FORMAT_TIME, &pos)) - d->m_position = pos; + std::optional<std::chrono::nanoseconds> pos = QGstElement::position(); + if (pos) + d->m_position = *pos; + else + qDebug() << "QGstPipeline: failed to query position, using previous position"; + return d->m_position; } -qint64 QGstPipeline::duration() const +std::chrono::milliseconds QGstPipeline::positionInMs() const { - gint64 d; - if (!gst_element_query_duration(element(), GST_FORMAT_TIME, &d)) - return 0.; - return d; + using namespace std::chrono; + return round<milliseconds>(position()); } QGstPipelinePrivate *QGstPipeline::getPrivate() const @@ -383,6 +412,3 @@ QGstPipelinePrivate *QGstPipeline::getPrivate() const } QT_END_NAMESPACE - -#include "qgstpipeline.moc" - diff --git a/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h b/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h index 23610dd00..ef08bfaaa 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h @@ -15,10 +15,10 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <QObject> +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtCore/qobject.h> -#include <common/qgst_p.h> +#include "qgst_p.h" QT_BEGIN_NAMESPACE @@ -73,6 +73,8 @@ public: GstPipeline *pipeline() const { return GST_PIPELINE_CAST(get()); } + void processMessages(GstMessageType = GST_MESSAGE_ANY); + void dumpGraph(const char *fileName); template <typename Functor> @@ -94,16 +96,18 @@ public: void flush(); - bool seek(qint64 pos, double rate); - bool setPlaybackRate(double rate); + void setPlaybackRate(double rate); double playbackRate() const; + void applyPlaybackRate(bool instantRateChange); - bool setPosition(qint64 pos); - qint64 position() const; - - qint64 duration() const; + void setPosition(std::chrono::nanoseconds pos); + std::chrono::nanoseconds position() const; + std::chrono::milliseconds positionInMs() const; private: + void seek(std::chrono::nanoseconds pos, double rate); + void seek(std::chrono::nanoseconds pos); + QGstPipelinePrivate *getPrivate() const; void beginConfig(); diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp index 0381b921e..7c620da39 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp @@ -21,24 +21,23 @@ QT_BEGIN_NAMESPACE QMaybe<QPlatformAudioInput *> QGstreamerAudioInput::create(QAudioInput *parent) { - QGstElement autoaudiosrc = QGstElement::createFromFactory("autoaudiosrc", "autoaudiosrc"); - if (!autoaudiosrc) - return errorMessageCannotFindElement("autoaudiosrc"); + static const auto error = qGstErrorMessageIfElementsNotAvailable("autoaudiosrc", "volume"); + if (error) + return *error; - QGstElement volume = QGstElement::createFromFactory("volume", "volume"); - if (!volume) - return errorMessageCannotFindElement("volume"); - - return new QGstreamerAudioInput(autoaudiosrc, volume, parent); + return new QGstreamerAudioInput(parent); } -QGstreamerAudioInput::QGstreamerAudioInput(QGstElement autoaudiosrc, QGstElement volume, - QAudioInput *parent) +QGstreamerAudioInput::QGstreamerAudioInput(QAudioInput *parent) : QObject(parent), QPlatformAudioInput(parent), gstAudioInput(QGstBin::create("audioInput")), - audioSrc(std::move(autoaudiosrc)), - audioVolume(std::move(volume)) + audioSrc{ + QGstElement::createFromFactory("autoaudiosrc", "autoaudiosrc"), + }, + audioVolume{ + QGstElement::createFromFactory("volume", "volume"), + } { gstAudioInput.add(audioSrc, audioVolume); qLinkGstElements(audioSrc, audioVolume); @@ -46,6 +45,56 @@ QGstreamerAudioInput::QGstreamerAudioInput(QGstElement autoaudiosrc, QGstElement gstAudioInput.addGhostPad(audioVolume, "src"); } +QGstElement QGstreamerAudioInput::createGstElement() +{ + const auto *customDeviceInfo = + dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(m_audioDevice.handle()); + + if (customDeviceInfo) { + qCDebug(qLcMediaAudioInput) + << "requesting custom audio src element: " << customDeviceInfo->id; + + QGstElement element = QGstBin::createFromPipelineDescription(customDeviceInfo->id, + /*name=*/nullptr, + /*ghostUnlinkedPads=*/true); + if (element) + return element; + + qCWarning(qLcMediaAudioInput) + << "Cannot create audio source element:" << customDeviceInfo->id; + } + + const QByteArray &id = m_audioDevice.id(); + if constexpr (QT_CONFIG(pulseaudio)) { + QGstElement newSrc = QGstElement::createFromFactory("pulsesrc", "audiosrc"); + if (newSrc) { + newSrc.set("device", id.constData()); + return newSrc; + } else { + qWarning() << "Cannot create pulsesrc"; + } + } else if constexpr (QT_CONFIG(alsa)) { + QGstElement newSrc = QGstElement::createFromFactory("alsasrc", "audiosrc"); + if (newSrc) { + newSrc.set("device", id.constData()); + return newSrc; + } else { + qWarning() << "Cannot create alsasrc"; + } + } else { + auto *deviceInfo = dynamic_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); + if (deviceInfo && deviceInfo->gstDevice) { + QGstElement element = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosrc"); + if (element) + return element; + } + } + qCWarning(qLcMediaAudioInput) << "Invalid audio device"; + qCWarning(qLcMediaAudioInput) + << "Failed to create a gst element for the audio device, using a default audio source"; + return QGstElement::createFromFactory("autoaudiosrc", "audiosrc"); +} + QGstreamerAudioInput::~QGstreamerAudioInput() { gstAudioInput.setStateSync(GST_STATE_NULL); @@ -68,26 +117,7 @@ void QGstreamerAudioInput::setAudioDevice(const QAudioDevice &device) qCDebug(qLcMediaAudioInput) << "setAudioInput" << device.description() << device.isNull(); m_audioDevice = device; - QGstElement newSrc; - if constexpr (QT_CONFIG(pulseaudio)) { - auto id = m_audioDevice.id(); - newSrc = QGstElement::createFromFactory("pulsesrc", "audiosrc"); - if (!newSrc.isNull()) - newSrc.set("device", id.constData()); - else - qCWarning(qLcMediaAudioInput) << "Invalid audio device"; - } else { - auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); - if (deviceInfo && deviceInfo->gstDevice) - newSrc = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosrc"); - else - qCWarning(qLcMediaAudioInput) << "Invalid audio device"; - } - - if (newSrc.isNull()) { - qCWarning(qLcMediaAudioInput) << "Failed to create a gst element for the audio device, using a default audio source"; - newSrc = QGstElement::createFromFactory("autoaudiosrc", "audiosrc"); - } + QGstElement newSrc = createGstElement(); QGstPipeline::modifyPipelineWhileNotRunning(gstAudioInput.getPipeline(), [&] { qUnlinkGstElements(audioSrc, audioVolume); diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h index 69500ecab..5ca0e1a49 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h @@ -44,7 +44,9 @@ public: QGstElement gstElement() const { return gstAudioInput; } private: - QGstreamerAudioInput(QGstElement autoaudiosrc, QGstElement volume, QAudioInput *parent); + explicit QGstreamerAudioInput(QAudioInput *parent); + + QGstElement createGstElement(); QAudioDevice m_audioDevice; diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp index f45c371e9..9cea7fb62 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp @@ -16,43 +16,90 @@ QT_BEGIN_NAMESPACE QMaybe<QPlatformAudioOutput *> QGstreamerAudioOutput::create(QAudioOutput *parent) { - QGstElement audioconvert = QGstElement::createFromFactory("audioconvert", "audioConvert"); - if (!audioconvert) - return errorMessageCannotFindElement("audioconvert"); + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "audioconvert", "audioresample", "volume", "autoaudiosink"); + if (error) + return *error; - QGstElement audioresample = QGstElement::createFromFactory("audioresample", "audioResample"); - if (!audioresample) - return errorMessageCannotFindElement("audioresample"); - - QGstElement volume = QGstElement::createFromFactory("volume", "volume"); - if (!volume) - return errorMessageCannotFindElement("volume"); - - QGstElement autoaudiosink = QGstElement::createFromFactory("autoaudiosink", "autoAudioSink"); - if (!autoaudiosink) - return errorMessageCannotFindElement("autoaudiosink"); - - return new QGstreamerAudioOutput(audioconvert, audioresample, volume, autoaudiosink, parent); + return new QGstreamerAudioOutput(parent); } -QGstreamerAudioOutput::QGstreamerAudioOutput(QGstElement audioconvert, QGstElement audioresample, - QGstElement volume, QGstElement autoaudiosink, - QAudioOutput *parent) +QGstreamerAudioOutput::QGstreamerAudioOutput(QAudioOutput *parent) : QObject(parent), QPlatformAudioOutput(parent), gstAudioOutput(QGstBin::create("audioOutput")), - audioConvert(std::move(audioconvert)), - audioResample(std::move(audioresample)), - audioVolume(std::move(volume)), - audioSink(std::move(autoaudiosink)) + audioQueue{ + QGstElement::createFromFactory("queue", "audioQueue"), + }, + audioConvert{ + QGstElement::createFromFactory("audioconvert", "audioConvert"), + }, + audioResample{ + QGstElement::createFromFactory("audioresample", "audioResample"), + }, + audioVolume{ + QGstElement::createFromFactory("volume", "volume"), + }, + audioSink{ + QGstElement::createFromFactory("autoaudiosink", "autoAudioSink"), + } { - audioQueue = QGstElement::createFromFactory("queue", "audioQueue"); gstAudioOutput.add(audioQueue, audioConvert, audioResample, audioVolume, audioSink); qLinkGstElements(audioQueue, audioConvert, audioResample, audioVolume, audioSink); gstAudioOutput.addGhostPad(audioQueue, "sink"); } +QGstElement QGstreamerAudioOutput::createGstElement() +{ + const auto *customDeviceInfo = + dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(m_audioOutput.handle()); + + if (customDeviceInfo) { + qCDebug(qLcMediaAudioOutput) + << "requesting custom audio sink element: " << customDeviceInfo->id; + + QGstElement element = + QGstBin::createFromPipelineDescription(customDeviceInfo->id, /*name=*/nullptr, + /*ghostUnlinkedPads=*/true); + if (element) + return element; + + qCWarning(qLcMediaAudioOutput) + << "Cannot create audio sink element:" << customDeviceInfo->id; + } + + const QByteArray &id = m_audioOutput.id(); + if constexpr (QT_CONFIG(pulseaudio)) { + QGstElement newSink = QGstElement::createFromFactory("pulsesink", "audiosink"); + if (newSink) { + newSink.set("device", id.constData()); + return newSink; + } else { + qWarning() << "Cannot create pulsesink"; + } + } else if constexpr (QT_CONFIG(alsa)) { + QGstElement newSink = QGstElement::createFromFactory("alsasink", "audiosink"); + if (newSink) { + newSink.set("device", id.constData()); + return newSink; + } else { + qWarning() << "Cannot create alsasink"; + } + } else { + auto *deviceInfo = dynamic_cast<const QGStreamerAudioDeviceInfo *>(m_audioOutput.handle()); + if (deviceInfo && deviceInfo->gstDevice) { + QGstElement element = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosink"); + if (element) + return element; + } + } + qCWarning(qLcMediaAudioOutput) << "Invalid audio device:" << m_audioOutput.id(); + qCWarning(qLcMediaAudioOutput) + << "Failed to create a gst element for the audio device, using a default audio sink"; + return QGstElement::createFromFactory("autoaudiosink", "audiosink"); +} + QGstreamerAudioOutput::~QGstreamerAudioOutput() { gstAudioOutput.setStateSync(GST_STATE_NULL); @@ -73,28 +120,10 @@ void QGstreamerAudioOutput::setAudioDevice(const QAudioDevice &info) if (info == m_audioOutput) return; qCDebug(qLcMediaAudioOutput) << "setAudioOutput" << info.description() << info.isNull(); - m_audioOutput = info; - QGstElement newSink; - if constexpr (QT_CONFIG(pulseaudio)) { - auto id = m_audioOutput.id(); - newSink = QGstElement::createFromFactory("pulsesink", "audiosink"); - if (!newSink.isNull()) - newSink.set("device", id.constData()); - else - qCWarning(qLcMediaAudioOutput) << "Invalid audio device"; - } else { - auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_audioOutput.handle()); - if (deviceInfo && deviceInfo->gstDevice) - newSink = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosink"); - else - qCWarning(qLcMediaAudioOutput) << "Invalid audio device"; - } + m_audioOutput = info; - if (newSink.isNull()) { - qCWarning(qLcMediaAudioOutput) << "Failed to create a gst element for the audio device, using a default audio sink"; - newSink = QGstElement::createFromFactory("autoaudiosink", "audiosink"); - } + QGstElement newSink = createGstElement(); QGstPipeline::modifyPipelineWhileNotRunning(gstAudioOutput.getPipeline(), [&] { qUnlinkGstElements(audioVolume, audioSink); diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h index 4b528d9ee..dea53e5c4 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h @@ -41,8 +41,9 @@ public: QGstElement gstElement() const { return gstAudioOutput; } private: - QGstreamerAudioOutput(QGstElement audioconvert, QGstElement audioresample, QGstElement volume, - QGstElement autoaudiosink, QAudioOutput *parent); + explicit QGstreamerAudioOutput(QAudioOutput *parent); + + QGstElement createGstElement(); QAudioDevice m_audioOutput; diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp index 341cb69b3..9cba810db 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp @@ -3,6 +3,8 @@ #include <common/qgstreamerbufferprobe_p.h> +#include <common/qgst_p.h> + QT_BEGIN_NAMESPACE QGstreamerBufferProbe::QGstreamerBufferProbe(Flags flags) @@ -14,10 +16,14 @@ QGstreamerBufferProbe::~QGstreamerBufferProbe() = default; void QGstreamerBufferProbe::addProbeToPad(GstPad *pad, bool downstream) { - if (GstCaps *caps = gst_pad_get_current_caps(pad)) { - probeCaps(caps); - gst_caps_unref(caps); - } + QGstCaps caps{ + gst_pad_get_current_caps(pad), + QGstCaps::HasRef, + }; + + if (caps) + probeCaps(caps.caps()); + if (m_flags & ProbeCaps) { m_capsProbeId = gst_pad_add_probe( pad, diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp index 8388ce8d2..ce5efb648 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp @@ -26,6 +26,10 @@ #include <sys/stat.h> #include <fcntl.h> +#if QT_CONFIG(gstreamer_gl) +# include <gst/gl/gl.h> +#endif + static Q_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player") QT_BEGIN_NAMESPACE @@ -74,13 +78,25 @@ QGstreamerMediaPlayer::TrackSelector &QGstreamerMediaPlayer::trackSelector(Track return ts; } +void QGstreamerMediaPlayer::updateBufferProgress(float newProgress) +{ + if (qFuzzyIsNull(newProgress - m_bufferProgress)) + return; + + m_bufferProgress = newProgress; + bufferProgressChanged(m_bufferProgress); +} + void QGstreamerMediaPlayer::disconnectDecoderHandlers() { auto handlers = std::initializer_list<QGObjectHandlerScopedConnection *>{ - &padAdded, &padRemoved, &sourceSetup, &elementAdded, &unknownType, + &padAdded, &padRemoved, &sourceSetup, &uridecodebinElementAdded, + &unknownType, &elementAdded, &elementRemoved, }; for (QGObjectHandlerScopedConnection *handler : handlers) handler->disconnect(); + + decodeBinQueues = 0; } QMaybe<QPlatformMediaPlayer *> QGstreamerMediaPlayer::create(QMediaPlayer *parent) @@ -89,35 +105,26 @@ QMaybe<QPlatformMediaPlayer *> QGstreamerMediaPlayer::create(QMediaPlayer *paren if (!videoOutput) return videoOutput.error(); - QGstElement videoInputSelector = - QGstElement::createFromFactory("input-selector", "videoInputSelector"); - if (!videoInputSelector) - return errorMessageCannotFindElement("input-selector"); + static const auto error = + qGstErrorMessageIfElementsNotAvailable("input-selector", "decodebin", "uridecodebin"); + if (error) + return *error; - QGstElement audioInputSelector = - QGstElement::createFromFactory("input-selector", "audioInputSelector"); - if (!audioInputSelector) - return errorMessageCannotFindElement("input-selector"); - - QGstElement subTitleInputSelector = - QGstElement::createFromFactory("input-selector", "subTitleInputSelector"); - if (!subTitleInputSelector) - return errorMessageCannotFindElement("input-selector"); - - return new QGstreamerMediaPlayer(videoOutput.value(), videoInputSelector, audioInputSelector, - subTitleInputSelector, parent); + return new QGstreamerMediaPlayer(videoOutput.value(), parent); } QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, - QGstElement videoInputSelector, - QGstElement audioInputSelector, - QGstElement subTitleInputSelector, QMediaPlayer *parent) : QObject(parent), QPlatformMediaPlayer(parent), - trackSelectors{ { { VideoStream, videoInputSelector }, - { AudioStream, audioInputSelector }, - { SubtitleStream, subTitleInputSelector } } }, + trackSelectors{ { + { VideoStream, + QGstElement::createFromFactory("input-selector", "videoInputSelector") }, + { AudioStream, + QGstElement::createFromFactory("input-selector", "audioInputSelector") }, + { SubtitleStream, + QGstElement::createFromFactory("input-selector", "subTitleInputSelector") }, + } }, playerPipeline(QGstPipeline::create("playerPipeline")), gstVideoOutput(videoOutput) { @@ -129,7 +136,6 @@ QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, for (auto &ts : trackSelectors) playerPipeline.add(ts.selector); - playerPipeline.setState(GST_STATE_NULL); playerPipeline.installMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this)); playerPipeline.installMessageFilter(static_cast<QGstreamerSyncMessageFilter *>(this)); @@ -139,7 +145,9 @@ QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, gst_pipeline_use_clock(playerPipeline.pipeline(), systemClock.get()); - connect(&positionUpdateTimer, &QTimer::timeout, this, &QGstreamerMediaPlayer::updatePosition); + connect(&positionUpdateTimer, &QTimer::timeout, this, [this] { + updatePositionFromPipeline(); + }); } QGstreamerMediaPlayer::~QGstreamerMediaPlayer() @@ -147,25 +155,45 @@ QGstreamerMediaPlayer::~QGstreamerMediaPlayer() playerPipeline.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this)); playerPipeline.removeMessageFilter(static_cast<QGstreamerSyncMessageFilter *>(this)); playerPipeline.setStateSync(GST_STATE_NULL); - topology.free(); } -qint64 QGstreamerMediaPlayer::position() const +std::chrono::nanoseconds QGstreamerMediaPlayer::pipelinePosition() const +{ + if (m_url.isEmpty()) + return {}; + + Q_ASSERT(playerPipeline); + return playerPipeline.position(); +} + +void QGstreamerMediaPlayer::updatePositionFromPipeline() { - if (playerPipeline.isNull() || m_url.isEmpty()) - return 0; + using namespace std::chrono; - return playerPipeline.position()/1e6; + positionChanged(round<milliseconds>(pipelinePosition())); +} + +void QGstreamerMediaPlayer::updateDurationFromPipeline() +{ + std::optional<std::chrono::milliseconds> duration = playerPipeline.durationInMs(); + if (!duration) + duration = std::chrono::milliseconds{ -1 }; + + if (duration != m_duration) { + qCDebug(qLcMediaPlayer) << "updateDurationFromPipeline" << *duration; + m_duration = *duration; + durationChanged(m_duration); + } } qint64 QGstreamerMediaPlayer::duration() const { - return m_duration; + return m_duration.count(); } float QGstreamerMediaPlayer::bufferProgress() const { - return m_bufferProgress/100.; + return m_bufferProgress; } QMediaTimeRange QGstreamerMediaPlayer::availablePlaybackRanges() const @@ -180,18 +208,28 @@ qreal QGstreamerMediaPlayer::playbackRate() const void QGstreamerMediaPlayer::setPlaybackRate(qreal rate) { - if (playerPipeline.setPlaybackRate(rate)) - playbackRateChanged(rate); + if (rate == m_rate) + return; + + m_rate = rate; + + playerPipeline.setPlaybackRate(rate); + playbackRateChanged(rate); } void QGstreamerMediaPlayer::setPosition(qint64 pos) { - qint64 currentPos = playerPipeline.position()/1e6; - if (pos == currentPos) + std::chrono::milliseconds posInMs{ pos }; + setPosition(posInMs); +} + +void QGstreamerMediaPlayer::setPosition(std::chrono::milliseconds pos) +{ + if (pos == playerPipeline.position()) return; playerPipeline.finishStateChange(); - playerPipeline.setPosition(pos*1e6); - qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << pos << playerPipeline.position()/1e6; + playerPipeline.setPosition(pos); + qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << pos << playerPipeline.positionInMs(); if (mediaStatus() == QMediaPlayer::EndOfMedia) mediaStatusChanged(QMediaPlayer::LoadedMedia); positionChanged(pos); @@ -201,12 +239,14 @@ void QGstreamerMediaPlayer::play() { if (state() == QMediaPlayer::PlayingState || m_url.isEmpty()) return; - resetCurrentLoop(); + + if (state() != QMediaPlayer::PausedState) + resetCurrentLoop(); playerPipeline.setInStoppedState(false); if (mediaStatus() == QMediaPlayer::EndOfMedia) { - playerPipeline.setPosition(0); - updatePosition(); + playerPipeline.setPosition({}); + positionChanged(0); } qCDebug(qLcMediaPlayer) << "play()."; @@ -216,13 +256,17 @@ void QGstreamerMediaPlayer::play() // immediately, when they happen while paused. playerPipeline.flush(); m_requiresSeekOnPlay = false; + } else { + // we get an assertion failure during instant playback rate changes + // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3545 + constexpr bool performInstantRateChange = false; + playerPipeline.applyPlaybackRate(/*instantRateChange=*/performInstantRateChange); } if (ret == GST_STATE_CHANGE_FAILURE) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the playing state."; - if (mediaStatus() == QMediaPlayer::LoadedMedia) - mediaStatusChanged(QMediaPlayer::BufferedMedia); - emit stateChanged(QMediaPlayer::PlayingState); + positionUpdateTimer.start(100); + stateChanged(QMediaPlayer::PlayingState); } void QGstreamerMediaPlayer::pause() @@ -236,53 +280,83 @@ void QGstreamerMediaPlayer::pause() playerPipeline.setInStoppedState(false); playerPipeline.flush(); } - int ret = playerPipeline.setState(GST_STATE_PAUSED); + int ret = playerPipeline.setStateSync(GST_STATE_PAUSED); if (ret == GST_STATE_CHANGE_FAILURE) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; if (mediaStatus() == QMediaPlayer::EndOfMedia) { - playerPipeline.setPosition(0); - mediaStatusChanged(QMediaPlayer::BufferedMedia); + playerPipeline.setPosition({}); + positionChanged(0); + } else { + updatePositionFromPipeline(); } - updatePosition(); - emit stateChanged(QMediaPlayer::PausedState); + stateChanged(QMediaPlayer::PausedState); + + if (m_bufferProgress > 0 || !canTrackProgress()) + mediaStatusChanged(QMediaPlayer::BufferedMedia); + else + mediaStatusChanged(QMediaPlayer::BufferingMedia); } void QGstreamerMediaPlayer::stop() { + using namespace std::chrono_literals; if (state() == QMediaPlayer::StoppedState) { if (position() != 0) { - playerPipeline.setPosition(0); - positionChanged(0); + playerPipeline.setPosition({}); + positionChanged(0ms); + mediaStatusChanged(QMediaPlayer::LoadedMedia); } return; } stopOrEOS(false); } -void *QGstreamerMediaPlayer::nativePipeline() +const QGstPipeline &QGstreamerMediaPlayer::pipeline() const { - return playerPipeline.pipeline(); + return playerPipeline; } void QGstreamerMediaPlayer::stopOrEOS(bool eos) { + using namespace std::chrono_literals; + positionUpdateTimer.stop(); playerPipeline.setInStoppedState(true); bool ret = playerPipeline.setStateSync(GST_STATE_PAUSED); if (!ret) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the stopped state."; - if (!eos) - playerPipeline.setPosition(0); - updatePosition(); - emit stateChanged(QMediaPlayer::StoppedState); - mediaStatusChanged(eos ? QMediaPlayer::EndOfMedia : QMediaPlayer::LoadedMedia); + if (!eos) { + playerPipeline.setPosition(0ms); + positionChanged(0ms); + } + stateChanged(QMediaPlayer::StoppedState); + if (eos) + mediaStatusChanged(QMediaPlayer::EndOfMedia); + else + mediaStatusChanged(QMediaPlayer::LoadedMedia); + m_initialBufferProgressSent = false; + bufferProgressChanged(0.f); } -bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) +void QGstreamerMediaPlayer::detectPipelineIsSeekable() { - if (message.isNull()) - return false; + qCDebug(qLcMediaPlayer) << "detectPipelineIsSeekable"; + QGstQueryHandle query{ + gst_query_new_seeking(GST_FORMAT_TIME), + QGstQueryHandle::HasRef, + }; + gboolean canSeek = false; + if (gst_element_query(playerPipeline.element(), query.get())) { + gst_query_parse_seeking(query.get(), nullptr, &canSeek, nullptr, nullptr); + qCDebug(qLcMediaPlayer) << " pipeline is seekable:" << canSeek; + } else { + qCWarning(qLcMediaPlayer) << " query for seekable failed."; + } + seekableChanged(canSeek); +} +bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) +{ qCDebug(qLcMediaPlayer) << "received bus message:" << message; GstMessage* gm = message.message(); @@ -293,34 +367,55 @@ bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) gst_message_parse_tag(gm, &tagList); qCDebug(qLcMediaPlayer) << " Got tags: " << tagList.get(); - auto metaData = QGstreamerMetaData::fromGstTagList(tagList.get()); - for (auto k : metaData.keys()) - m_metaData.insert(k, metaData.value(k)); + + QMediaMetaData originalMetaData = m_metaData; + extendMetaDataFromTagList(m_metaData, tagList); + if (originalMetaData != m_metaData) + metaDataChanged(); + + if (gstVideoOutput) { + QVariant rotation = m_metaData.value(QMediaMetaData::Orientation); + gstVideoOutput->setRotation(rotation.value<QtVideo::Rotation>()); + } break; } case GST_MESSAGE_DURATION_CHANGED: { - qint64 d = playerPipeline.duration()/1e6; - qCDebug(qLcMediaPlayer) << " duration changed message" << d; - if (d != m_duration) { - m_duration = d; - emit durationChanged(duration()); - } + if (!prerolling) + updateDurationFromPipeline(); + return false; } - case GST_MESSAGE_EOS: + case GST_MESSAGE_EOS: { + positionChanged(m_duration); if (doLoop()) { setPosition(0); break; } stopOrEOS(true); break; + } case GST_MESSAGE_BUFFERING: { - qCDebug(qLcMediaPlayer) << " buffering message"; int progress = 0; gst_message_parse_buffering(gm, &progress); - m_bufferProgress = progress; - mediaStatusChanged(m_bufferProgress == 100 ? QMediaPlayer::BufferedMedia : QMediaPlayer::BufferingMedia); - emit bufferProgressChanged(m_bufferProgress/100.); + + qCDebug(qLcMediaPlayer) << " buffering message: " << progress; + + if (state() != QMediaPlayer::StoppedState && !prerolling) { + if (!m_initialBufferProgressSent) { + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + } + + if (m_bufferProgress > 0 && progress == 0) + mediaStatusChanged(QMediaPlayer::StalledMedia); + else if (progress >= 50) + // QTBUG-124517: rethink buffering + mediaStatusChanged(QMediaPlayer::BufferedMedia); + else + mediaStatusChanged(QMediaPlayer::BufferingMedia); + } + + updateBufferProgress(progress * 0.01); break; } case GST_MESSAGE_STATE_CHANGED: { @@ -332,78 +427,81 @@ bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) GstState pending; gst_message_parse_state_changed(gm, &oldState, &newState, &pending); - qCDebug(qLcMediaPlayer) << " state changed message from" << oldState << "to" << newState - << pending; + qCDebug(qLcMediaPlayer) << " state changed message from" + << QCompactGstMessageAdaptor(message); switch (newState) { case GST_STATE_VOID_PENDING: case GST_STATE_NULL: case GST_STATE_READY: break; - case GST_STATE_PAUSED: - { + case GST_STATE_PAUSED: { if (prerolling) { qCDebug(qLcMediaPlayer) << "Preroll done, setting status to Loaded"; - prerolling = false; - GST_DEBUG_BIN_TO_DOT_FILE(playerPipeline.bin(), GST_DEBUG_GRAPH_SHOW_ALL, "playerPipeline"); + playerPipeline.dumpGraph("playerPipelinePrerollDone"); - qint64 d = playerPipeline.duration()/1e6; - if (d != m_duration) { - m_duration = d; - qCDebug(qLcMediaPlayer) << " duration changed" << d; - emit durationChanged(duration()); - } + prerolling = false; + updateDurationFromPipeline(); + m_metaData.insert(QMediaMetaData::Duration, duration()); + m_metaData.insert(QMediaMetaData::Url, m_url); parseStreamsAndMetadata(); + metaDataChanged(); - emit tracksChanged(); + tracksChanged(); mediaStatusChanged(QMediaPlayer::LoadedMedia); - GstQuery *query = gst_query_new_seeking(GST_FORMAT_TIME); - gboolean canSeek = false; - if (gst_element_query(playerPipeline.element(), query)) { - gst_query_parse_seeking(query, nullptr, &canSeek, nullptr, nullptr); - qCDebug(qLcMediaPlayer) << " pipeline is seekable:" << canSeek; - } else { - qCDebug(qLcMediaPlayer) << " query for seekable failed."; + if (!playerPipeline.inStoppedState()) { + Q_ASSERT(!m_initialBufferProgressSent); + + bool immediatelySendBuffered = !canTrackProgress() || m_bufferProgress > 0; + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + if (immediatelySendBuffered) + mediaStatusChanged(QMediaPlayer::BufferedMedia); } - gst_query_unref(query); - seekableChanged(canSeek); } break; } - case GST_STATE_PLAYING: - mediaStatusChanged(QMediaPlayer::BufferedMedia); + case GST_STATE_PLAYING: { + if (!m_initialBufferProgressSent) { + bool immediatelySendBuffered = !canTrackProgress() || m_bufferProgress > 0; + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + if (immediatelySendBuffered) + mediaStatusChanged(QMediaPlayer::BufferedMedia); + } break; } + } break; } case GST_MESSAGE_ERROR: { + qCDebug(qLcMediaPlayer) << " error" << QCompactGstMessageAdaptor(message); + QUniqueGErrorHandle err; QUniqueGStringHandle debug; gst_message_parse_error(gm, &err, &debug); - qCDebug(qLcMediaPlayer) << " error" << err << debug; - GQuark errorDomain = err.get()->domain; gint errorCode = err.get()->code; if (errorDomain == GST_STREAM_ERROR) { if (errorCode == GST_STREAM_ERROR_CODEC_NOT_FOUND) - emit error(QMediaPlayer::FormatError, tr("Cannot play stream of type: <unknown>")); + error(QMediaPlayer::FormatError, tr("Cannot play stream of type: <unknown>")); else { - emit error(QMediaPlayer::FormatError, QString::fromUtf8(err.get()->message)); + error(QMediaPlayer::FormatError, QString::fromUtf8(err.get()->message)); } } else if (errorDomain == GST_RESOURCE_ERROR) { if (errorCode == GST_RESOURCE_ERROR_NOT_FOUND) { if (m_resourceErrorState != ResourceErrorState::ErrorReported) { // gstreamer seems to deliver multiple GST_RESOURCE_ERROR_NOT_FOUND events - emit error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); + error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); m_resourceErrorState = ResourceErrorState::ErrorReported; m_url.clear(); } } else { - emit error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); + error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); } } else { playerPipeline.dumpGraph("error"); @@ -411,40 +509,40 @@ bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) mediaStatusChanged(QMediaPlayer::InvalidMedia); break; } - case GST_MESSAGE_WARNING: { - QUniqueGErrorHandle err; - QUniqueGStringHandle debug; - gst_message_parse_warning (gm, &err, &debug); - qCWarning(qLcMediaPlayer) << "Warning:" << err; + + case GST_MESSAGE_WARNING: + qCWarning(qLcMediaPlayer) << "Warning:" << QCompactGstMessageAdaptor(message); playerPipeline.dumpGraph("warning"); break; - } - case GST_MESSAGE_INFO: { - if (qLcMediaPlayer().isDebugEnabled()) { - QUniqueGErrorHandle err; - QUniqueGStringHandle debug; - gst_message_parse_info (gm, &err, &debug); - qCDebug(qLcMediaPlayer) << "Info:" << err; - } + + case GST_MESSAGE_INFO: + if (qLcMediaPlayer().isDebugEnabled()) + qCDebug(qLcMediaPlayer) << "Info:" << QCompactGstMessageAdaptor(message); break; - } + case GST_MESSAGE_SEGMENT_START: { qCDebug(qLcMediaPlayer) << " segment start message, updating position"; - QGstStructure structure(gst_message_get_structure(gm)); + QGstStructureView structure(gst_message_get_structure(gm)); auto p = structure["position"].toInt64(); if (p) { - qint64 position = (*p)/1000000; - emit positionChanged(position); + std::chrono::milliseconds position{ + (*p) / 1000000, + }; + positionChanged(position); } break; } case GST_MESSAGE_ELEMENT: { - QGstStructure structure(gst_message_get_structure(gm)); + QGstStructureView structure(gst_message_get_structure(gm)); auto type = structure.name(); - if (type == "stream-topology") { - topology.free(); - topology = structure.copy(); - } + if (type == "stream-topology") + topology = structure.clone(); + + break; + } + + case GST_MESSAGE_ASYNC_DONE: { + detectPipelineIsSeekable(); break; } @@ -524,19 +622,19 @@ void QGstreamerMediaPlayer::decoderPadAdded(const QGstElement &src, const QGstPa if (streamType == VideoStream) { connectOutput(ts); ts.setActiveInputPad(sinkPad); - emit videoAvailableChanged(true); + videoAvailableChanged(true); } else if (streamType == AudioStream) { connectOutput(ts); ts.setActiveInputPad(sinkPad); - emit audioAvailableChanged(true); + audioAvailableChanged(true); } } if (!prerolling) - emit tracksChanged(); + tracksChanged(); - decoderOutputMap.insert(pad.name(), sinkPad); + decoderOutputMap.emplace(pad, sinkPad); } void QGstreamerMediaPlayer::decoderPadRemoved(const QGstElement &src, const QGstPad &pad) @@ -545,9 +643,11 @@ void QGstreamerMediaPlayer::decoderPadRemoved(const QGstElement &src, const QGst return; qCDebug(qLcMediaPlayer) << "Removed pad" << pad.name() << "from" << src.name(); - auto track = decoderOutputMap.value(pad.name()); - if (track.isNull()) + + auto it = decoderOutputMap.find(pad); + if (it == decoderOutputMap.end()) return; + QGstPad track = it->second; auto ts = std::find_if(std::begin(trackSelectors), std::end(trackSelectors), [&](TrackSelector &ts){ return ts.selector == track.parent(); }); @@ -604,7 +704,7 @@ void QGstreamerMediaPlayer::connectOutput(TrackSelector &ts) qCDebug(qLcMediaPlayer) << "connecting output for track type" << ts.type; playerPipeline.add(e); qLinkGstElements(ts.selector, e); - e.setState(GST_STATE_PAUSED); + e.syncStateWithParent(); } ts.isConnected = true; @@ -639,6 +739,18 @@ void QGstreamerMediaPlayer::removeOutput(TrackSelector &ts) ts.isConnected = false; } +void QGstreamerMediaPlayer::removeDynamicPipelineElements() +{ + for (QGstElement *element : { &src, &decoder }) { + if (element->isNull()) + continue; + + element->setStateSync(GstState::GST_STATE_NULL); + playerPipeline.remove(*element); + *element = QGstElement{}; + } +} + void QGstreamerMediaPlayer::uridecodebinElementAddedCallback(GstElement * /*uridecodebin*/, GstElement *child, QGstreamerMediaPlayer *) @@ -647,9 +759,7 @@ void QGstreamerMediaPlayer::uridecodebinElementAddedCallback(GstElement * /*urid qCDebug(qLcMediaPlayer) << "New element added to uridecodebin:" << c.name(); static const GType decodeBinType = [] { - QGstElementFactoryHandle factory = QGstElementFactoryHandle{ - gst_element_factory_find("decodebin"), - }; + QGstElementFactoryHandle factory = QGstElement::findFactory("decodebin"); return gst_element_factory_get_element_type(factory.get()); }(); @@ -705,8 +815,43 @@ void QGstreamerMediaPlayer::unknownTypeCallback(GstElement *decodebin, GstPad *p }); } +static bool isQueue(const QGstElement &element) +{ + static const GType queueType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("queue"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + static const GType multiQueueType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("multiqueue"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + return element.type() == queueType || element.type() == multiQueueType; +} + +void QGstreamerMediaPlayer::decodebinElementAddedCallback(GstBin * /*decodebin*/, + GstBin * /*sub_bin*/, GstElement *child, + QGstreamerMediaPlayer *self) +{ + QGstElement c(child, QGstElement::NeedsRef); + if (isQueue(c)) + self->decodeBinQueues += 1; +} + +void QGstreamerMediaPlayer::decodebinElementRemovedCallback(GstBin * /*decodebin*/, + GstBin * /*sub_bin*/, GstElement *child, + QGstreamerMediaPlayer *self) +{ + QGstElement c(child, QGstElement::NeedsRef); + if (isQueue(c)) + self->decodeBinQueues -= 1; +} + void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) { + using namespace std::chrono_literals; + qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << "setting location to" << content; prerolling = true; @@ -719,30 +864,27 @@ void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) m_url = content; m_stream = stream; - if (!src.isNull()) - playerPipeline.remove(src); - if (!decoder.isNull()) - playerPipeline.remove(decoder); - src = QGstElement(); + removeDynamicPipelineElements(); disconnectDecoderHandlers(); - decoder = QGstElement(); removeAllOutputs(); seekableChanged(false); Q_ASSERT(playerPipeline.inStoppedState()); - if (m_duration != 0) { - m_duration = 0; - durationChanged(0); + if (m_duration != 0ms) { + m_duration = 0ms; + durationChanged(0ms); } stateChanged(QMediaPlayer::StoppedState); if (position() != 0) - positionChanged(0); - mediaStatusChanged(QMediaPlayer::NoMedia); + positionChanged(0ms); if (!m_metaData.isEmpty()) { m_metaData.clear(); metaDataChanged(); } + if (content.isEmpty() && !stream) + mediaStatusChanged(QMediaPlayer::NoMedia); + if (content.isEmpty()) return; @@ -752,20 +894,23 @@ void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) if (maybeAppSrc) { m_appSrc = maybeAppSrc.value(); } else { - emit error(QMediaPlayer::ResourceError, maybeAppSrc.error()); + error(QMediaPlayer::ResourceError, maybeAppSrc.error()); return; } } src = m_appSrc->element(); decoder = QGstElement::createFromFactory("decodebin", "decoder"); if (!decoder) { - emit error(QMediaPlayer::ResourceError, errorMessageCannotFindElement("decodebin")); + error(QMediaPlayer::ResourceError, qGstErrorMessageCannotFindElement("decodebin")); return; } decoder.set("post-stream-topology", true); decoder.set("use-buffering", true); - unknownType = decoder.connect("unknown-type", - GCallback(QGstreamerMediaPlayer::unknownTypeCallback), this); + unknownType = decoder.connect("unknown-type", GCallback(unknownTypeCallback), this); + elementAdded = decoder.connect("deep-element-added", + GCallback(decodebinElementAddedCallback), this); + elementRemoved = decoder.connect("deep-element-removed", + GCallback(decodebinElementAddedCallback), this); playerPipeline.add(src, decoder); qLinkGstElements(src, decoder); @@ -776,7 +921,7 @@ void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) // use uridecodebin decoder = QGstElement::createFromFactory("uridecodebin", "decoder"); if (!decoder) { - emit error(QMediaPlayer::ResourceError, errorMessageCannotFindElement("uridecodebin")); + error(QMediaPlayer::ResourceError, qGstErrorMessageCannotFindElement("uridecodebin")); return; } playerPipeline.add(decoder); @@ -787,34 +932,38 @@ void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) } else { // can't set post-stream-topology to true, as uridecodebin doesn't have the property. // Use a hack - elementAdded = decoder.connect( - "element-added", - GCallback(QGstreamerMediaPlayer::uridecodebinElementAddedCallback), this); + uridecodebinElementAdded = decoder.connect( + "element-added", GCallback(uridecodebinElementAddedCallback), this); } - sourceSetup = decoder.connect("source-setup", - GCallback(QGstreamerMediaPlayer::sourceSetupCallback), this); - - unknownType = decoder.connect("unknown-type", - GCallback(QGstreamerMediaPlayer::unknownTypeCallback), this); + sourceSetup = decoder.connect("source-setup", GCallback(sourceSetupCallback), this); + unknownType = decoder.connect("unknown-type", GCallback(unknownTypeCallback), this); decoder.set("uri", content.toEncoded().constData()); decoder.set("use-buffering", true); - if (m_bufferProgress != 0) { - m_bufferProgress = 0; - emit bufferProgressChanged(0.); - } + + constexpr int mb = 1024 * 1024; + decoder.set("ring-buffer-max-size", 2 * mb); + + updateBufferProgress(0.f); + + elementAdded = decoder.connect("deep-element-added", + GCallback(decodebinElementAddedCallback), this); + elementRemoved = decoder.connect("deep-element-removed", + GCallback(decodebinElementAddedCallback), this); } padAdded = decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAdded>(this); padRemoved = decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemoved>(this); mediaStatusChanged(QMediaPlayer::LoadingMedia); - - if (!playerPipeline.setState(GST_STATE_PAUSED)) + if (!playerPipeline.setStateSync(GST_STATE_PAUSED)) { qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; + // Note: no further error handling: errors will be delivered via a GstMessage + return; + } - playerPipeline.setPosition(0); - positionChanged(0); + playerPipeline.setPosition(0ms); + positionChanged(0ms); } void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) @@ -844,9 +993,9 @@ void QGstreamerMediaPlayer::setVideoSink(QVideoSink *sink) gstVideoOutput->setVideoSink(sink); } -static QGstStructure endOfChain(const QGstStructure &s) +static QGstStructureView endOfChain(const QGstStructureView &s) { - QGstStructure e = s; + QGstStructureView e = s; while (1) { auto next = e["next"].toStructure(); if (!next.isNull()) @@ -860,32 +1009,26 @@ static QGstStructure endOfChain(const QGstStructure &s) void QGstreamerMediaPlayer::parseStreamsAndMetadata() { qCDebug(qLcMediaPlayer) << "============== parse topology ============"; - if (topology.isNull()) { + + if (!topology) { qCDebug(qLcMediaPlayer) << " null topology"; return; } - auto caps = topology["caps"].toCaps(); - auto structure = caps.at(0); - auto fileFormat = QGstreamerFormatInfo::fileFormatForCaps(structure); - qCDebug(qLcMediaPlayer) << caps << fileFormat; - m_metaData.insert(QMediaMetaData::FileFormat, QVariant::fromValue(fileFormat)); - m_metaData.insert(QMediaMetaData::Duration, duration()); - m_metaData.insert(QMediaMetaData::Url, m_url); - QGValue tags = topology["tags"]; - if (!tags.isNull()) { - QGstTagListHandle tagList; - gst_structure_get(topology.structure, "tags", GST_TYPE_TAG_LIST, &tagList, nullptr); - const auto metaData = QGstreamerMetaData::fromGstTagList(tagList.get()); - for (auto k : metaData.keys()) - m_metaData.insert(k, metaData.value(k)); - } + QGstStructureView topologyView{ topology }; - auto demux = endOfChain(topology); - auto next = demux["next"]; + QGstCaps caps = topologyView.caps(); + extendMetaDataFromCaps(m_metaData, caps); + + QGstTagListHandle tagList = QGstStructureView{ topology }.tags(); + if (tagList) + extendMetaDataFromTagList(m_metaData, tagList); + + QGstStructureView demux = endOfChain(topologyView); + QGValue next = demux["next"]; if (!next.isList()) { qCDebug(qLcMediaPlayer) << " no additional streams"; - emit metaDataChanged(); + metaDataChanged(); return; } @@ -893,43 +1036,28 @@ void QGstreamerMediaPlayer::parseStreamsAndMetadata() int size = next.listSize(); for (int i = 0; i < size; ++i) { auto val = next.at(i); - caps = val.toStructure()["caps"].toCaps(); - structure = caps.at(0); - if (structure.name().startsWith("audio/")) { - auto codec = QGstreamerFormatInfo::audioCodecForCaps(structure); - m_metaData.insert(QMediaMetaData::AudioCodec, QVariant::fromValue(codec)); - qCDebug(qLcMediaPlayer) << " audio" << caps << (int)codec; - } else if (structure.name().startsWith("video/")) { - auto codec = QGstreamerFormatInfo::videoCodecForCaps(structure); - m_metaData.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(codec)); - qCDebug(qLcMediaPlayer) << " video" << caps << (int)codec; - auto framerate = structure["framerate"].getFraction(); - if (framerate) - m_metaData.insert(QMediaMetaData::VideoFrameRate, *framerate); - - QSize resolution = structure.resolution(); - if (resolution.isValid()) - m_metaData.insert(QMediaMetaData::Resolution, resolution); + caps = val.toStructure().caps(); + + extendMetaDataFromCaps(m_metaData, caps); + + QGstStructureView structure = caps.at(0); + + if (structure.name().startsWith("video/")) { + QSize nativeSize = structure.nativeSize(); + gstVideoOutput->setNativeSize(nativeSize); } } auto sinkPad = trackSelector(VideoStream).activeInputPad(); - if (!sinkPad.isNull()) { - QGstTagListHandle tagList; - - g_object_get(sinkPad.object(), "tags", &tagList, nullptr); + if (sinkPad) { + QGstTagListHandle tagList = sinkPad.tags(); if (tagList) qCDebug(qLcMediaPlayer) << " tags=" << tagList.get(); else qCDebug(qLcMediaPlayer) << " tags=(null)"; - - QSize nativeSize = structure.nativeSize(); - gstVideoOutput->setNativeSize(nativeSize); } - qCDebug(qLcMediaPlayer) << "============== end parse topology ============"; - emit metaDataChanged(); playerPipeline.dumpGraph("playback"); } @@ -941,13 +1069,11 @@ int QGstreamerMediaPlayer::trackCount(QPlatformMediaPlayer::TrackType type) QMediaMetaData QGstreamerMediaPlayer::trackMetaData(QPlatformMediaPlayer::TrackType type, int index) { auto track = trackSelector(type).inputPad(index); - if (track.isNull()) + if (!track) return {}; - QGstTagListHandle tagList; - g_object_get(track.object(), "tags", &tagList, nullptr); - - return tagList ? QGstreamerMetaData::fromGstTagList(tagList.get()) : QMediaMetaData{}; + QGstTagListHandle tagList = track.tags(); + return taglistToMetaData(tagList); } int QGstreamerMediaPlayer::activeTrack(TrackType type) @@ -986,5 +1112,3 @@ void QGstreamerMediaPlayer::setActiveTrack(TrackType type, int index) } QT_END_NAMESPACE - -#include "moc_qgstreamermediaplayer_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h index 3b2129296..28e7a0c31 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h @@ -40,13 +40,10 @@ class QGstreamerMediaPlayer : public QObject, public QGstreamerBusMessageFilter, public QGstreamerSyncMessageFilter { - Q_OBJECT - public: static QMaybe<QPlatformMediaPlayer *> create(QMediaPlayer *parent = nullptr); ~QGstreamerMediaPlayer(); - qint64 position() const override; qint64 duration() const override; float bufferProgress() const override; @@ -58,7 +55,7 @@ public: QUrl media() const override; const QIODevice *mediaStream() const override; - void setMedia(const QUrl&, QIODevice *) override; + void setMedia(const QUrl &, QIODevice *) override; bool streamPlaybackSupported() const override { return true; } @@ -74,24 +71,22 @@ public: void setActiveTrack(TrackType, int /*streamNumber*/) override; void setPosition(qint64 pos) override; + void setPosition(std::chrono::milliseconds pos); void play() override; void pause() override; void stop() override; - void *nativePipeline() override; + const QGstPipeline &pipeline() const; bool processBusMessage(const QGstreamerMessage& message) override; bool processSyncMessage(const QGstreamerMessage& message) override; -public Q_SLOTS: - void updatePosition() { positionChanged(position()); } private: - QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, QGstElement videoInputSelector, - QGstElement audioInputSelector, QGstElement subTitleInputSelector, - QMediaPlayer *parent); + QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, QMediaPlayer *parent); - struct TrackSelector { + struct TrackSelector + { TrackSelector(TrackType, QGstElement selector); QGstPad createInputPad(); void removeInputPad(QGstPad pad); @@ -115,22 +110,36 @@ private: void decoderPadAdded(const QGstElement &src, const QGstPad &pad); void decoderPadRemoved(const QGstElement &src, const QGstPad &pad); void disconnectDecoderHandlers(); - static void uridecodebinElementAddedCallback(GstElement *uridecodebin, GstElement *child, QGstreamerMediaPlayer *that); - static void sourceSetupCallback(GstElement *uridecodebin, GstElement *source, QGstreamerMediaPlayer *that); + static void uridecodebinElementAddedCallback(GstElement *uridecodebin, GstElement *child, + QGstreamerMediaPlayer *that); + static void sourceSetupCallback(GstElement *uridecodebin, GstElement *source, + QGstreamerMediaPlayer *that); static void unknownTypeCallback(GstElement *decodebin, GstPad *pad, GstCaps *caps, QGstreamerMediaPlayer *self); + static void decodebinElementAddedCallback(GstBin *decodebin, GstBin *sub_bin, + GstElement *element, QGstreamerMediaPlayer *self); + static void decodebinElementRemovedCallback(GstBin *decodebin, GstBin *sub_bin, + GstElement *element, QGstreamerMediaPlayer *self); + void parseStreamsAndMetadata(); void connectOutput(TrackSelector &ts); void removeOutput(TrackSelector &ts); + void removeDynamicPipelineElements(); void removeAllOutputs(); void stopOrEOS(bool eos); + bool canTrackProgress() const { return decodeBinQueues > 0; } + void detectPipelineIsSeekable(); + + std::chrono::nanoseconds pipelinePosition() const; + void updatePositionFromPipeline(); + void updateDurationFromPipeline(); + void updateBufferProgress(float); std::array<TrackSelector, NTrackTypes> trackSelectors; TrackSelector &trackSelector(TrackType type); QMediaMetaData m_metaData; - int m_bufferProgress = 0; QUrl m_url; QIODevice *m_stream = nullptr; @@ -142,13 +151,16 @@ private: bool prerolling = false; bool m_requiresSeekOnPlay = false; + bool m_initialBufferProgressSent = false; ResourceErrorState m_resourceErrorState = ResourceErrorState::NoError; - qint64 m_duration = 0; + float m_rate = 1.f; + float m_bufferProgress = 0.f; + std::chrono::milliseconds m_duration{}; QTimer positionUpdateTimer; QGstAppSource *m_appSrc = nullptr; - QGstStructure topology; + QUniqueGstStructureHandle topology; // Gst elements QGstPipeline playerPipeline; @@ -160,14 +172,26 @@ private: // QGstElement streamSynchronizer; - QHash<QByteArray, QGstPad> decoderOutputMap; + struct QGstPadLess + { + bool operator()(const QGstPad &lhs, const QGstPad &rhs) const + { + return lhs.pad() < rhs.pad(); + } + }; + + std::map<QGstPad, QGstPad, QGstPadLess> decoderOutputMap; // decoder connections QGObjectHandlerScopedConnection padAdded; QGObjectHandlerScopedConnection padRemoved; QGObjectHandlerScopedConnection sourceSetup; - QGObjectHandlerScopedConnection elementAdded; + QGObjectHandlerScopedConnection uridecodebinElementAdded; QGObjectHandlerScopedConnection unknownType; + QGObjectHandlerScopedConnection elementAdded; + QGObjectHandlerScopedConnection elementRemoved; + + int decodeBinQueues = 0; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h index 01fe68acb..9836bd0cb 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h @@ -43,7 +43,7 @@ public: GstMessageType type() const { return GST_MESSAGE_TYPE(get()); } QGstObject source() const { return QGstObject(GST_MESSAGE_SRC(get()), QGstObject::NeedsRef); } - QGstStructure structure() const { return QGstStructure(gst_message_get_structure(get())); } + QGstStructureView structure() const { return QGstStructureView(gst_message_get_structure(get())); } GstMessage *message() const { return get(); } }; diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp index 953acb56a..9aa9406b9 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp @@ -3,6 +3,7 @@ #include "qgstreamermetadata_p.h" #include <QtMultimedia/qmediametadata.h> +#include <QtMultimedia/qtvideo.h> #include <QtCore/qdebug.h> #include <QtCore/qdatetime.h> #include <QtCore/qlocale.h> @@ -12,267 +13,477 @@ #include <gst/gstversion.h> #include <common/qgst_handle_types_p.h> #include <common/qgstutils_p.h> +#include <qgstreamerformatinfo_p.h> QT_BEGIN_NAMESPACE -struct { +namespace { + +namespace MetadataLookupImpl { + +#ifdef __cpp_lib_constexpr_algorithms +# define constexpr_lookup constexpr +#else +# define constexpr_lookup /*constexpr*/ +#endif + +struct MetadataKeyValuePair +{ const char *tag; QMediaMetaData::Key key; -} gstTagToMetaDataKey[] = { - { GST_TAG_TITLE, QMediaMetaData::Title }, - { GST_TAG_COMMENT, QMediaMetaData::Comment }, - { GST_TAG_DESCRIPTION, QMediaMetaData::Description }, - { GST_TAG_GENRE, QMediaMetaData::Genre }, - { GST_TAG_DATE_TIME, QMediaMetaData::Date }, - { GST_TAG_DATE, QMediaMetaData::Date }, +}; + +constexpr const char *toTag(const char *t) +{ + return t; +} +constexpr const char *toTag(const MetadataKeyValuePair &kv) +{ + return kv.tag; +} + +constexpr QMediaMetaData::Key toKey(QMediaMetaData::Key k) +{ + return k; +} +constexpr QMediaMetaData::Key toKey(const MetadataKeyValuePair &kv) +{ + return kv.key; +} + +constexpr auto compareByKey = [](const auto &lhs, const auto &rhs) { + return toKey(lhs) < toKey(rhs); +}; - { GST_TAG_LANGUAGE_CODE, QMediaMetaData::Language }, +constexpr auto compareByTag = [](const auto &lhs, const auto &rhs) { + return std::strcmp(toTag(lhs), toTag(rhs)) < 0; +}; + +constexpr_lookup auto makeLookupTable() +{ + std::array<MetadataKeyValuePair, 22> lookupTable{ { + { GST_TAG_TITLE, QMediaMetaData::Title }, + { GST_TAG_COMMENT, QMediaMetaData::Comment }, + { GST_TAG_DESCRIPTION, QMediaMetaData::Description }, + { GST_TAG_GENRE, QMediaMetaData::Genre }, + { GST_TAG_DATE_TIME, QMediaMetaData::Date }, + { GST_TAG_DATE, QMediaMetaData::Date }, + + { GST_TAG_LANGUAGE_CODE, QMediaMetaData::Language }, + + { GST_TAG_ORGANIZATION, QMediaMetaData::Publisher }, + { GST_TAG_COPYRIGHT, QMediaMetaData::Copyright }, + + // Media + { GST_TAG_DURATION, QMediaMetaData::Duration }, + + // Audio + { GST_TAG_BITRATE, QMediaMetaData::AudioBitRate }, + { GST_TAG_AUDIO_CODEC, QMediaMetaData::AudioCodec }, + + // Music + { GST_TAG_ALBUM, QMediaMetaData::AlbumTitle }, + { GST_TAG_ALBUM_ARTIST, QMediaMetaData::AlbumArtist }, + { GST_TAG_ARTIST, QMediaMetaData::ContributingArtist }, + { GST_TAG_TRACK_NUMBER, QMediaMetaData::TrackNumber }, + + { GST_TAG_PREVIEW_IMAGE, QMediaMetaData::ThumbnailImage }, + { GST_TAG_IMAGE, QMediaMetaData::CoverArtImage }, + + // Image/Video + { "resolution", QMediaMetaData::Resolution }, + { GST_TAG_IMAGE_ORIENTATION, QMediaMetaData::Orientation }, + + // Video + { GST_TAG_VIDEO_CODEC, QMediaMetaData::VideoCodec }, + + // Movie + { GST_TAG_PERFORMER, QMediaMetaData::LeadPerformer }, + } }; + + std::sort(lookupTable.begin(), lookupTable.end(), + [](const MetadataKeyValuePair &lhs, const MetadataKeyValuePair &rhs) { + return std::string_view(lhs.tag) < std::string_view(rhs.tag); + }); + return lookupTable; +} - { GST_TAG_ORGANIZATION, QMediaMetaData::Publisher }, - { GST_TAG_COPYRIGHT, QMediaMetaData::Copyright }, +constexpr_lookup auto gstTagToMetaDataKey = makeLookupTable(); +constexpr_lookup auto metaDataKeyToGstTag = [] { + auto array = gstTagToMetaDataKey; + std::sort(array.begin(), array.end(), compareByKey); + return array; +}(); - // Media - { GST_TAG_DURATION, QMediaMetaData::Duration }, +} // namespace MetadataLookupImpl - // Audio - { GST_TAG_BITRATE, QMediaMetaData::AudioBitRate }, - { GST_TAG_AUDIO_CODEC, QMediaMetaData::AudioCodec }, +QMediaMetaData::Key tagToKey(const char *tag) +{ + if (tag == nullptr) + return QMediaMetaData::Key(-1); - // Music - { GST_TAG_ALBUM, QMediaMetaData::AlbumTitle }, - { GST_TAG_ALBUM_ARTIST, QMediaMetaData::AlbumArtist }, - { GST_TAG_ARTIST, QMediaMetaData::ContributingArtist }, - { GST_TAG_TRACK_NUMBER, QMediaMetaData::TrackNumber }, + using namespace MetadataLookupImpl; + auto foundIterator = std::lower_bound(gstTagToMetaDataKey.begin(), gstTagToMetaDataKey.end(), + tag, compareByTag); + if (std::strcmp(foundIterator->tag, tag) == 0) + return foundIterator->key; - { GST_TAG_PREVIEW_IMAGE, QMediaMetaData::ThumbnailImage }, - { GST_TAG_IMAGE, QMediaMetaData::CoverArtImage }, + return QMediaMetaData::Key(-1); +} - // Image/Video - { "resolution", QMediaMetaData::Resolution }, - { GST_TAG_IMAGE_ORIENTATION, QMediaMetaData::Orientation }, +const char *keyToTag(QMediaMetaData::Key key) +{ + using namespace MetadataLookupImpl; + auto foundIterator = std::lower_bound(metaDataKeyToGstTag.begin(), metaDataKeyToGstTag.end(), + key, compareByKey); + if (foundIterator->key == key) + return foundIterator->tag; - // Video - { GST_TAG_VIDEO_CODEC, QMediaMetaData::VideoCodec }, + return nullptr; +} - // Movie - { GST_TAG_PERFORMER, QMediaMetaData::LeadPerformer }, +#undef constexpr_lookup - { nullptr, QMediaMetaData::Title } -}; +QtVideo::Rotation parseRotationTag(const char *string) +{ + using namespace std::string_view_literals; + + if (string == "rotate-90"sv) + return QtVideo::Rotation::Clockwise90; + if (string == "rotate-180"sv) + return QtVideo::Rotation::Clockwise180; + if (string == "rotate-270"sv) + return QtVideo::Rotation::Clockwise270; + if (string == "rotate-0"sv) + return QtVideo::Rotation::None; + + qCritical() << "cannot parse orientation: {}" << string; + return QtVideo::Rotation::None; +} + +QDateTime parseDate(const GValue &val) +{ + Q_ASSERT(G_VALUE_TYPE(&val) == G_TYPE_DATE); + + const GDate *date = (const GDate *)g_value_get_boxed(&val); + if (!g_date_valid(date)) + return {}; + + int year = g_date_get_year(date); + int month = g_date_get_month(date); + int day = g_date_get_day(date); + return QDateTime(QDate(year, month, day), QTime()); +} -static QMediaMetaData::Key tagToKey(const char *tag) +QDateTime parseDateTime(const GValue &val) { - auto *map = gstTagToMetaDataKey; - while (map->tag) { - if (!strcmp(map->tag, tag)) - return map->key; - ++map; + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_DATE_TIME); + + const GstDateTime *dateTime = (const GstDateTime *)g_value_get_boxed(&val); + int year = gst_date_time_has_year(dateTime) ? gst_date_time_get_year(dateTime) : 0; + int month = gst_date_time_has_month(dateTime) ? gst_date_time_get_month(dateTime) : 0; + int day = gst_date_time_has_day(dateTime) ? gst_date_time_get_day(dateTime) : 0; + int hour = 0; + int minute = 0; + int second = 0; + float tz = 0; + if (gst_date_time_has_time(dateTime)) { + hour = gst_date_time_get_hour(dateTime); + minute = gst_date_time_get_minute(dateTime); + second = gst_date_time_get_second(dateTime); + tz = gst_date_time_get_time_zone_offset(dateTime); } - return QMediaMetaData::Key(-1); + return QDateTime{ + QDate(year, month, day), + QTime(hour, minute, second), + QTimeZone(tz * 60 * 60), + }; } -static const char *keyToTag(QMediaMetaData::Key key) +QImage parseImage(const GValue &val) { - auto *map = gstTagToMetaDataKey; - while (map->tag) { - if (map->key == key) - return map->tag; - ++map; + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_SAMPLE); + + GstSample *sample = (GstSample *)g_value_get_boxed(&val); + GstCaps *caps = gst_sample_get_caps(sample); + if (caps && !gst_caps_is_empty(caps)) { + GstStructure *structure = gst_caps_get_structure(caps, 0); + const gchar *name = gst_structure_get_name(structure); + if (QByteArray(name).startsWith("image/")) { + GstBuffer *buffer = gst_sample_get_buffer(sample); + if (buffer) { + GstMapInfo info; + gst_buffer_map(buffer, &info, GST_MAP_READ); + QImage image = QImage::fromData(info.data, info.size, name); + gst_buffer_unmap(buffer, &info); + return image; + } + } + } + + return {}; +} + +std::optional<double> parseFractionAsDouble(const GValue &val) +{ + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_FRACTION); + + int nom = gst_value_get_fraction_numerator(&val); + int denom = gst_value_get_fraction_denominator(&val); + if (denom == 0) + return std::nullopt; + return double(nom) / double(denom); +} + +constexpr std::string_view extendedComment{ GST_TAG_EXTENDED_COMMENT }; + +void addTagsFromExtendedComment(const GstTagList *list, const gchar *tag, QMediaMetaData &metadata) +{ + using namespace Qt::Literals; + assert(tag == extendedComment); + + int entryCount = gst_tag_list_get_tag_size(list, tag); + for (int i = 0; i != entryCount; ++i) { + const GValue *value = gst_tag_list_get_value_index(list, tag, i); + + const QLatin1StringView strValue{ g_value_get_string(value) }; + + auto equalIndex = strValue.indexOf(QLatin1StringView("=")); + if (equalIndex == -1) { + qDebug() << "Cannot parse GST_TAG_EXTENDED_COMMENT entry: " << value; + continue; + } + + const QLatin1StringView key = strValue.first(equalIndex); + const QLatin1StringView valueString = strValue.last(strValue.size() - equalIndex - 1); + + if (key == "DURATION"_L1) { + QUniqueGstDateTimeHandle duration{ + gst_date_time_new_from_iso8601_string(valueString.data()), + }; + + if (duration) { + using namespace std::chrono; + + auto chronoDuration = hours(gst_date_time_get_hour(duration.get())) + + minutes(gst_date_time_get_minute(duration.get())) + + seconds(gst_date_time_get_second(duration.get())) + + microseconds(gst_date_time_get_microsecond(duration.get())); + + metadata.insert(QMediaMetaData::Duration, + QVariant::fromValue(round<milliseconds>(chronoDuration).count())); + } + } } - return nullptr; } -//internal -static void addTagToMap(const GstTagList *list, - const gchar *tag, - gpointer user_data) +void addTagToMetaData(const GstTagList *list, const gchar *tag, void *userdata) { + QMediaMetaData &metadata = *reinterpret_cast<QMediaMetaData *>(userdata); + QMediaMetaData::Key key = tagToKey(tag); - if (key == QMediaMetaData::Key(-1)) - return; + if (key == QMediaMetaData::Key(-1)) { + if (tag == extendedComment) + addTagsFromExtendedComment(list, tag, metadata); - auto *map = reinterpret_cast<QHash<QMediaMetaData::Key, QVariant>* >(user_data); + return; + } - GValue val; - val.g_type = 0; + GValue val{}; gst_tag_list_copy_value(&val, list, tag); - switch (G_VALUE_TYPE(&val)) { - case G_TYPE_STRING: { + GType type = G_VALUE_TYPE(&val); + + if (auto entryCount = gst_tag_list_get_tag_size(list, tag) != 0; entryCount != 1) + qWarning() << "addTagToMetaData: invaled entry count for" << tag << "-" << entryCount; + + if (type == G_TYPE_STRING) { const gchar *str_value = g_value_get_string(&val); - if (key == QMediaMetaData::Language) { - map->insert(key, - QVariant::fromValue(QLocale::codeToLanguage(QString::fromUtf8(str_value), - QLocale::ISO639Part2))); + + switch (key) { + case QMediaMetaData::Language: { + metadata.insert(key, + QVariant::fromValue(QLocale::codeToLanguage( + QString::fromUtf8(str_value), QLocale::AnyLanguageCode))); break; } - map->insert(key, QString::fromUtf8(str_value)); - break; - } - case G_TYPE_INT: - map->insert(key, g_value_get_int(&val)); - break; - case G_TYPE_UINT: - map->insert(key, g_value_get_uint(&val)); - break; - case G_TYPE_LONG: - map->insert(key, qint64(g_value_get_long(&val))); - break; - case G_TYPE_BOOLEAN: - map->insert(key, g_value_get_boolean(&val)); - break; - case G_TYPE_CHAR: - map->insert(key, g_value_get_schar(&val)); - break; - case G_TYPE_DOUBLE: - map->insert(key, g_value_get_double(&val)); - break; - default: - // GST_TYPE_DATE is a function, not a constant, so pull it out of the switch - if (G_VALUE_TYPE(&val) == G_TYPE_DATE) { - const GDate *date = (const GDate *)g_value_get_boxed(&val); - if (g_date_valid(date)) { - int year = g_date_get_year(date); - int month = g_date_get_month(date); - int day = g_date_get_day(date); - // don't insert if we already have a datetime. - if (!map->contains(key)) - map->insert(key, QDateTime(QDate(year, month, day), QTime())); - } - } else if (G_VALUE_TYPE(&val) == GST_TYPE_DATE_TIME) { - const GstDateTime *dateTime = (const GstDateTime *)g_value_get_boxed(&val); - int year = gst_date_time_has_year(dateTime) ? gst_date_time_get_year(dateTime) : 0; - int month = gst_date_time_has_month(dateTime) ? gst_date_time_get_month(dateTime) : 0; - int day = gst_date_time_has_day(dateTime) ? gst_date_time_get_day(dateTime) : 0; - int hour = 0; - int minute = 0; - int second = 0; - float tz = 0; - if (gst_date_time_has_time(dateTime)) { - hour = gst_date_time_get_hour(dateTime); - minute = gst_date_time_get_minute(dateTime); - second = gst_date_time_get_second(dateTime); - tz = gst_date_time_get_time_zone_offset(dateTime); - } - QDateTime qDateTime(QDate(year, month, day), QTime(hour, minute, second), - QTimeZone(tz * 60 * 60)); - map->insert(key, qDateTime); - } else if (G_VALUE_TYPE(&val) == GST_TYPE_SAMPLE) { - GstSample *sample = (GstSample *)g_value_get_boxed(&val); - GstCaps *caps = gst_sample_get_caps(sample); - if (caps && !gst_caps_is_empty(caps)) { - GstStructure *structure = gst_caps_get_structure(caps, 0); - const gchar *name = gst_structure_get_name(structure); - if (QByteArray(name).startsWith("image/")) { - GstBuffer *buffer = gst_sample_get_buffer(sample); - if (buffer) { - GstMapInfo info; - gst_buffer_map(buffer, &info, GST_MAP_READ); - map->insert(key, QImage::fromData(info.data, info.size, name)); - gst_buffer_unmap(buffer, &info); - } - } - } - } else if (G_VALUE_TYPE(&val) == GST_TYPE_FRACTION) { - int nom = gst_value_get_fraction_numerator(&val); - int denom = gst_value_get_fraction_denominator(&val); - - if (denom > 0) { - map->insert(key, double(nom) / denom); - } + case QMediaMetaData::Orientation: { + metadata.insert(key, QVariant::fromValue(parseRotationTag(str_value))); + break; + } + default: + metadata.insert(key, QString::fromUtf8(str_value)); + break; + }; + } else if (type == G_TYPE_INT) { + metadata.insert(key, g_value_get_int(&val)); + } else if (type == G_TYPE_UINT) { + metadata.insert(key, g_value_get_uint(&val)); + } else if (type == G_TYPE_LONG) { + metadata.insert(key, qint64(g_value_get_long(&val))); + } else if (type == G_TYPE_BOOLEAN) { + metadata.insert(key, g_value_get_boolean(&val)); + } else if (type == G_TYPE_CHAR) { + metadata.insert(key, g_value_get_schar(&val)); + } else if (type == G_TYPE_DOUBLE) { + metadata.insert(key, g_value_get_double(&val)); + } else if (type == G_TYPE_DATE) { + if (!metadata.keys().contains(key)) { + QDateTime date = parseDate(val); + if (date.isValid()) + metadata.insert(key, date); } - break; + } else if (type == GST_TYPE_DATE_TIME) { + metadata.insert(key, parseDateTime(val)); + } else if (type == GST_TYPE_SAMPLE) { + QImage image = parseImage(val); + if (!image.isNull()) + metadata.insert(key, image); + } else if (type == GST_TYPE_FRACTION) { + std::optional<double> fraction = parseFractionAsDouble(val); + + if (fraction) + metadata.insert(key, *fraction); } g_value_unset(&val); } +} // namespace -QGstreamerMetaData QGstreamerMetaData::fromGstTagList(const GstTagList *tags) +QMediaMetaData taglistToMetaData(const QGstTagListHandle &handle) { - QGstreamerMetaData m; - gst_tag_list_foreach(tags, addTagToMap, &m.data); + QMediaMetaData m; + extendMetaDataFromTagList(m, handle); return m; } - -void QGstreamerMetaData::setMetaData(GstElement *element) const +void extendMetaDataFromTagList(QMediaMetaData &metadata, const QGstTagListHandle &handle) { - if (!GST_IS_TAG_SETTER(element)) - return; + if (handle) + gst_tag_list_foreach(handle.get(), reinterpret_cast<GstTagForeachFunc>(&addTagToMetaData), + &metadata); +} - gst_tag_setter_reset_tags(GST_TAG_SETTER(element)); +static void applyMetaDataToTagSetter(const QMediaMetaData &metadata, GstTagSetter *element) +{ + gst_tag_setter_reset_tags(element); - for (auto it = data.cbegin(), end = data.cend(); it != end; ++it) { - const char *tagName = keyToTag(it.key()); + for (QMediaMetaData::Key key : metadata.keys()) { + const char *tagName = keyToTag(key); if (!tagName) continue; - const QVariant &tagValue = it.value(); + const QVariant &tagValue = metadata.value(key); + + auto setTag = [&](const auto &value) { + gst_tag_setter_add_tags(element, GST_TAG_MERGE_REPLACE, tagName, value, nullptr); + }; switch (tagValue.typeId()) { - case QMetaType::QString: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toString().toUtf8().constData(), - nullptr); - break; - case QMetaType::Int: - case QMetaType::LongLong: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toInt(), - nullptr); - break; - case QMetaType::Double: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toDouble(), - nullptr); - break; - case QMetaType::QDate: - case QMetaType::QDateTime: { - QDateTime date = tagValue.toDateTime(); - - QGstGstDateTimeHandle dateTime{ - gst_date_time_new(date.offsetFromUtc() / 60. / 60., date.date().year(), - date.date().month(), date.date().day(), date.time().hour(), - date.time().minute(), date.time().second()), - QGstGstDateTimeHandle::HasRef, - }; - - gst_tag_setter_add_tags(GST_TAG_SETTER(element), GST_TAG_MERGE_REPLACE, tagName, - dateTime.get(), nullptr); - break; - } - default: { - if (tagValue.typeId() == qMetaTypeId<QLocale::Language>()) { - QByteArray language = QLocale::languageToCode(tagValue.value<QLocale::Language>(), QLocale::ISO639Part2).toUtf8(); - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - language.constData(), - nullptr); - } - - break; + case QMetaType::QString: + setTag(tagValue.toString().toUtf8().constData()); + break; + case QMetaType::Int: + case QMetaType::LongLong: + setTag(tagValue.toInt()); + break; + case QMetaType::Double: + setTag(tagValue.toDouble()); + break; + case QMetaType::QDate: + case QMetaType::QDateTime: { + QDateTime date = tagValue.toDateTime(); + + QGstGstDateTimeHandle dateTime{ + gst_date_time_new(date.offsetFromUtc() / 60. / 60., date.date().year(), + date.date().month(), date.date().day(), date.time().hour(), + date.time().minute(), date.time().second()), + QGstGstDateTimeHandle::HasRef, + }; + + setTag(dateTime.get()); + break; + } + default: { + if (tagValue.typeId() == qMetaTypeId<QLocale::Language>()) { + QByteArray language = QLocale::languageToCode(tagValue.value<QLocale::Language>(), + QLocale::ISO639Part2) + .toUtf8(); + setTag(language.constData()); } + + break; + } } } } -void QGstreamerMetaData::setMetaData(GstBin *bin) const +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstElement &element) { - GstIterator *elements = gst_bin_iterate_all_by_interface(bin, GST_TYPE_TAG_SETTER); - GValue item = G_VALUE_INIT; + GstTagSetter *tagSetter = qGstSafeCast<GstTagSetter>(element.element()); + if (tagSetter) + applyMetaDataToTagSetter(metadata, tagSetter); + else + qWarning() << "applyMetaDataToTagSetter failed: element not a GstTagSetter" + << element.name(); +} + +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstBin &bin) +{ + GstIterator *elements = gst_bin_iterate_all_by_interface(bin.bin(), GST_TYPE_TAG_SETTER); + GValue item = {}; + while (gst_iterator_next(elements, &item) == GST_ITERATOR_OK) { - GstElement * const element = GST_ELEMENT(g_value_get_object(&item)); - setMetaData(element); + GstElement *element = static_cast<GstElement *>(g_value_get_object(&item)); + if (!element) + continue; + + GstTagSetter *tagSetter = qGstSafeCast<GstTagSetter>(element); + + if (tagSetter) + applyMetaDataToTagSetter(metadata, tagSetter); } + gst_iterator_free(elements); } +void extendMetaDataFromCaps(QMediaMetaData &metadata, const QGstCaps &caps) +{ + QGstStructureView structure = caps.at(0); + + QMediaFormat::FileFormat fileFormat = QGstreamerFormatInfo::fileFormatForCaps(structure); + if (fileFormat != QMediaFormat::FileFormat::UnspecifiedFormat) { + // Container caps + metadata.insert(QMediaMetaData::FileFormat, fileFormat); + return; + } + + QMediaFormat::AudioCodec audioCodec = QGstreamerFormatInfo::audioCodecForCaps(structure); + if (audioCodec != QMediaFormat::AudioCodec::Unspecified) { + // Audio stream caps + metadata.insert(QMediaMetaData::AudioCodec, QVariant::fromValue(audioCodec)); + return; + } + + QMediaFormat::VideoCodec videoCodec = QGstreamerFormatInfo::videoCodecForCaps(structure); + if (videoCodec != QMediaFormat::VideoCodec::Unspecified) { + // Video stream caps + metadata.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(videoCodec)); + std::optional<float> framerate = structure["framerate"].getFraction(); + if (framerate) + metadata.insert(QMediaMetaData::VideoFrameRate, *framerate); + + QSize resolution = structure.resolution(); + if (resolution.isValid()) + metadata.insert(QMediaMetaData::Resolution, resolution); + } +} + +QMediaMetaData capsToMetaData(const QGstCaps &caps) +{ + QMediaMetaData metadata; + extendMetaDataFromCaps(metadata, caps); + return metadata; +} QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h index 7ff5552b2..f04a9aba9 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h @@ -16,20 +16,19 @@ // #include <qmediametadata.h> -#include <qvariant.h> -#include <gst/gst.h> +#include "qgst_p.h" QT_BEGIN_NAMESPACE -class QGstreamerMetaData : public QMediaMetaData -{ -public: - static QGstreamerMetaData fromGstTagList(const GstTagList *tags); +QMediaMetaData taglistToMetaData(const QGstTagListHandle &); +void extendMetaDataFromTagList(QMediaMetaData &, const QGstTagListHandle &); - void setMetaData(GstBin *bin) const; - void setMetaData(GstElement *element) const; -}; +QMediaMetaData capsToMetaData(const QGstCaps &); +void extendMetaDataFromCaps(QMediaMetaData &, const QGstCaps &); + +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstBin &); +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstElement &); QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp index 053dd973b..ef991f5b4 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp @@ -14,82 +14,83 @@ static Q_LOGGING_CATEGORY(qLcMediaVideoOutput, "qt.multimedia.videooutput") QT_BEGIN_NAMESPACE +static QGstElement makeVideoConvertScale(const char *name) +{ + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + if (factory) // videoconvertscale is only available in gstreamer 1.20 + return QGstElement::createFromFactory(factory, name); + + return QGstBin::createFromPipelineDescription("videoconvert ! videoscale", name, + /*ghostUnlinkedPads=*/true); +} + QMaybe<QGstreamerVideoOutput *> QGstreamerVideoOutput::create(QObject *parent) { - QGstElement videoConvert; - QGstElement videoScale; + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); - QGstElementFactoryHandle factory = QGstElementFactoryHandle{ - gst_element_factory_find("videoconvertscale"), - }; + static std::optional<QString> elementCheck = []() -> std::optional<QString> { + std::optional<QString> error = qGstErrorMessageIfElementsNotAvailable("fakesink", "queue"); + if (error) + return error; - if (factory) { // videoconvertscale is only available in gstreamer 1.20 - videoConvert = QGstElement::createFromFactory(factory, "videoConvertScale"); - } else { - videoConvert = QGstElement::createFromFactory("videoconvert", "videoConvert"); - if (!videoConvert) - return errorMessageCannotFindElement("videoconvert"); + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + if (factory) + return std::nullopt; - videoScale = QGstElement::createFromFactory("videoscale", "videoScale"); - if (!videoScale) - return errorMessageCannotFindElement("videoscale"); - } + return qGstErrorMessageIfElementsNotAvailable("videoconvert", "videoscale"); + }(); - QGstElement videoSink = QGstElement::createFromFactory("fakesink", "fakeVideoSink"); - if (!videoSink) - return errorMessageCannotFindElement("fakesink"); - videoSink.set("sync", true); + if (elementCheck) + return *elementCheck; - return new QGstreamerVideoOutput(videoConvert, videoScale, videoSink, parent); + return new QGstreamerVideoOutput(parent); } -QGstreamerVideoOutput::QGstreamerVideoOutput(QGstElement convert, QGstElement scale, - QGstElement sink, QObject *parent) +QGstreamerVideoOutput::QGstreamerVideoOutput(QObject *parent) : QObject(parent), - gstVideoOutput(QGstBin::create("videoOutput")), - videoConvert(std::move(convert)), - videoScale(std::move(scale)), - videoSink(std::move(sink)) + m_outputBin(QGstBin::create("videoOutput")), + m_videoQueue{ + QGstElement::createFromFactory("queue", "videoQueue"), + }, + m_videoConvertScale{ + makeVideoConvertScale("videoConvertScale"), + }, + m_videoSink{ + QGstElement::createFromFactory("fakesink", "fakeVideoSink"), + } { - videoQueue = QGstElement::createFromFactory("queue", "videoQueue"); - - videoSink.set("sync", true); - videoSink.set("async", false); // no asynchronous state changes + m_videoSink.set("sync", true); + m_videoSink.set("async", false); // no asynchronous state changes - if (videoScale) { - gstVideoOutput.add(videoQueue, videoConvert, videoScale, videoSink); - qLinkGstElements(videoQueue, videoConvert, videoScale, videoSink); - } else { - gstVideoOutput.add(videoQueue, videoConvert, videoSink); - qLinkGstElements(videoQueue, videoConvert, videoSink); - } + m_outputBin.add(m_videoQueue, m_videoConvertScale, m_videoSink); + qLinkGstElements(m_videoQueue, m_videoConvertScale, m_videoSink); - gstVideoOutput.addGhostPad(videoQueue, "sink"); + m_outputBin.addGhostPad(m_videoQueue, "sink"); } QGstreamerVideoOutput::~QGstreamerVideoOutput() { - gstVideoOutput.setStateSync(GST_STATE_NULL); + m_outputBin.setStateSync(GST_STATE_NULL); } void QGstreamerVideoOutput::setVideoSink(QVideoSink *sink) { auto *gstVideoSink = sink ? static_cast<QGstreamerVideoSink *>(sink->platformVideoSink()) : nullptr; - if (gstVideoSink == m_videoSink) + if (gstVideoSink == m_platformVideoSink) return; - if (m_videoSink) - m_videoSink->setPipeline({}); + if (m_platformVideoSink) + m_platformVideoSink->setPipeline({}); - m_videoSink = gstVideoSink; - if (m_videoSink) { - m_videoSink->setPipeline(gstPipeline); - if (nativeSize.isValid()) - m_videoSink->setNativeSize(nativeSize); + m_platformVideoSink = gstVideoSink; + if (m_platformVideoSink) { + m_platformVideoSink->setPipeline(m_pipeline); + if (m_nativeSize.isValid()) + m_platformVideoSink->setNativeSize(m_nativeSize); } QGstElement gstSink; - if (m_videoSink) { - gstSink = m_videoSink->gstSink(); + if (m_platformVideoSink) { + gstSink = m_platformVideoSink->gstSink(); } else { gstSink = QGstElement::createFromFactory("fakesink", "fakevideosink"); Q_ASSERT(gstSink); @@ -97,103 +98,104 @@ void QGstreamerVideoOutput::setVideoSink(QVideoSink *sink) gstSink.set("async", false); // no asynchronous state changes } - if (videoSink == gstSink) + if (m_videoSink == gstSink) return; - gstPipeline.modifyPipelineWhileNotRunning([&] { - if (!videoSink.isNull()) - gstVideoOutput.stopAndRemoveElements(videoSink); + m_pipeline.modifyPipelineWhileNotRunning([&] { + if (!m_videoSink.isNull()) + m_outputBin.stopAndRemoveElements(m_videoSink); - videoSink = gstSink; - gstVideoOutput.add(videoSink); + m_videoSink = gstSink; + m_outputBin.add(m_videoSink); - if (videoScale) - qLinkGstElements(videoScale, videoSink); - else - qLinkGstElements(videoConvert, videoSink); + qLinkGstElements(m_videoConvertScale, m_videoSink); GstEvent *event = gst_event_new_reconfigure(); - gst_element_send_event(videoSink.element(), event); - videoSink.syncStateWithParent(); + gst_element_send_event(m_videoSink.element(), event); + m_videoSink.syncStateWithParent(); doLinkSubtitleStream(); }); qCDebug(qLcMediaVideoOutput) << "sinkChanged" << gstSink.name(); - GST_DEBUG_BIN_TO_DOT_FILE(gstPipeline.bin(), - GstDebugGraphDetails(/*GST_DEBUG_GRAPH_SHOW_ALL |*/ GST_DEBUG_GRAPH_SHOW_MEDIA_TYPE | - GST_DEBUG_GRAPH_SHOW_NON_DEFAULT_PARAMS | GST_DEBUG_GRAPH_SHOW_STATES), - videoSink.name()); - + m_pipeline.dumpGraph(m_videoSink.name().constData()); } void QGstreamerVideoOutput::setPipeline(const QGstPipeline &pipeline) { - gstPipeline = pipeline; - if (m_videoSink) - m_videoSink->setPipeline(gstPipeline); + m_pipeline = pipeline; + if (m_platformVideoSink) + m_platformVideoSink->setPipeline(m_pipeline); } void QGstreamerVideoOutput::linkSubtitleStream(QGstElement src) { qCDebug(qLcMediaVideoOutput) << "link subtitle stream" << src.isNull(); - if (src == subtitleSrc) + if (src == m_subtitleSrc) return; - gstPipeline.modifyPipelineWhileNotRunning([&] { - subtitleSrc = src; + m_pipeline.modifyPipelineWhileNotRunning([&] { + m_subtitleSrc = src; doLinkSubtitleStream(); }); } void QGstreamerVideoOutput::unlinkSubtitleStream() { - if (subtitleSrc.isNull()) + if (m_subtitleSrc.isNull()) return; qCDebug(qLcMediaVideoOutput) << "unlink subtitle stream"; - subtitleSrc = {}; - if (!subtitleSink.isNull()) { - gstPipeline.modifyPipelineWhileNotRunning([&] { - gstPipeline.stopAndRemoveElements(subtitleSink); + m_subtitleSrc = {}; + if (!m_subtitleSink.isNull()) { + m_pipeline.modifyPipelineWhileNotRunning([&] { + m_pipeline.stopAndRemoveElements(m_subtitleSink); return; }); - subtitleSink = {}; + m_subtitleSink = {}; } - if (m_videoSink) - m_videoSink->setSubtitleText({}); + if (m_platformVideoSink) + m_platformVideoSink->setSubtitleText({}); } void QGstreamerVideoOutput::doLinkSubtitleStream() { - if (!subtitleSink.isNull()) { - gstPipeline.stopAndRemoveElements(subtitleSink); - subtitleSink = {}; + if (!m_subtitleSink.isNull()) { + m_pipeline.stopAndRemoveElements(m_subtitleSink); + m_subtitleSink = {}; } - if (!m_videoSink || subtitleSrc.isNull()) + if (!m_platformVideoSink || m_subtitleSrc.isNull()) return; - if (subtitleSink.isNull()) { - subtitleSink = m_videoSink->subtitleSink(); - gstPipeline.add(subtitleSink); + if (m_subtitleSink.isNull()) { + m_subtitleSink = m_platformVideoSink->subtitleSink(); + m_pipeline.add(m_subtitleSink); } - qLinkGstElements(subtitleSrc, subtitleSink); + qLinkGstElements(m_subtitleSrc, m_subtitleSink); +} + +void QGstreamerVideoOutput::updateNativeSize() +{ + if (!m_platformVideoSink) + return; + + m_platformVideoSink->setNativeSize(qRotatedFrameSize(m_nativeSize, m_rotation)); } void QGstreamerVideoOutput::setIsPreview() { // configures the queue to be fast and lightweight for camera preview // also avoids blocking the queue in case we have an encodebin attached to the tee as well - videoQueue.set("leaky", 2 /*downstream*/); - videoQueue.set("silent", true); - videoQueue.set("max-size-buffers", uint(1)); - videoQueue.set("max-size-bytes", uint(0)); - videoQueue.set("max-size-time", quint64(0)); + m_videoQueue.set("leaky", 2 /*downstream*/); + m_videoQueue.set("silent", true); + m_videoQueue.set("max-size-buffers", uint(1)); + m_videoQueue.set("max-size-bytes", uint(0)); + m_videoQueue.set("max-size-time", quint64(0)); } void QGstreamerVideoOutput::flushSubtitles() { - if (!subtitleSink.isNull()) { - auto pad = subtitleSink.staticPad("sink"); + if (!m_subtitleSink.isNull()) { + auto pad = m_subtitleSink.staticPad("sink"); auto *event = gst_event_new_flush_start(); pad.sendEvent(event); event = gst_event_new_flush_stop(false); @@ -203,9 +205,14 @@ void QGstreamerVideoOutput::flushSubtitles() void QGstreamerVideoOutput::setNativeSize(QSize sz) { - nativeSize = sz; - if (m_videoSink) - m_videoSink->setNativeSize(nativeSize); + m_nativeSize = sz; + updateNativeSize(); +} + +void QGstreamerVideoOutput::setRotation(QtVideo::Rotation rot) +{ + m_rotation = rot; + updateNativeSize(); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h index 62bd4b219..10d2f3ee7 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h @@ -38,11 +38,11 @@ public: ~QGstreamerVideoOutput(); void setVideoSink(QVideoSink *sink); - QGstreamerVideoSink *gstreamerVideoSink() const { return m_videoSink; } + QGstreamerVideoSink *gstreamerVideoSink() const { return m_platformVideoSink; } void setPipeline(const QGstPipeline &pipeline); - QGstElement gstElement() const { return gstVideoOutput; } + QGstElement gstElement() const { return m_outputBin; } void linkSubtitleStream(QGstElement subtitleSrc); void unlinkSubtitleStream(); @@ -50,28 +50,29 @@ public: void flushSubtitles(); void setNativeSize(QSize); + void setRotation(QtVideo::Rotation); private: - QGstreamerVideoOutput(QGstElement videoConvert, QGstElement videoScale, QGstElement videoSink, - QObject *parent); + explicit QGstreamerVideoOutput(QObject *parent); void doLinkSubtitleStream(); + void updateNativeSize(); - QPointer<QGstreamerVideoSink> m_videoSink; + QPointer<QGstreamerVideoSink> m_platformVideoSink; // Gst elements - QGstPipeline gstPipeline; + QGstPipeline m_pipeline; - QGstBin gstVideoOutput; - QGstElement videoQueue; - QGstElement videoConvert; - QGstElement videoScale; - QGstElement videoSink; + QGstBin m_outputBin; + QGstElement m_videoQueue; + QGstElement m_videoConvertScale; + QGstElement m_videoSink; - QGstElement subtitleSrc; - QGstElement subtitleSink; + QGstElement m_subtitleSrc; + QGstElement m_subtitleSink; - QSize nativeSize; + QSize m_nativeSize; + QtVideo::Rotation m_rotation{}; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp index 2ed2acb36..39377265a 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp @@ -37,10 +37,13 @@ QT_BEGIN_NAMESPACE static Q_LOGGING_CATEGORY(qLcGstVideoSink, "qt.multimedia.gstvideosink"); QGstreamerVideoSink::QGstreamerVideoSink(QVideoSink *parent) - : QPlatformVideoSink(parent) + : QPlatformVideoSink{ + parent, + }, + m_sinkBin{ + QGstBin::create("videoSinkBin"), + } { - sinkBin = QGstBin::create("videoSinkBin"); - // This is a hack for some iMX and NVidia platforms. These require the use of a special video // conversion element in the pipeline before the video sink, as they unfortunately // output some proprietary format from the decoder even though it's sometimes marked as @@ -48,38 +51,44 @@ QGstreamerVideoSink::QGstreamerVideoSink(QVideoSink *parent) // // To fix this, simply insert the element into the pipeline if it's available. Otherwise // we simply use an identity element. - gstQueue = QGstElement::createFromFactory("queue", "videoSinkQueue"); - QGstElementFactoryHandle factory; - // QT_MULTIMEDIA_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the + // QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the // conversion element. Ideally we construct the element programatically, though. - QByteArray preprocessOverride = - qgetenv("QT_MULTIMEDIA_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT"); + QByteArray preprocessOverride = qgetenv("QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT"); if (!preprocessOverride.isEmpty()) { - qCDebug(qLcGstVideoSink) << "requesting conversion element from environment: " + qCDebug(qLcGstVideoSink) << "requesting conversion element from environment:" << preprocessOverride; - factory = QGstElementFactoryHandle{ - gst_element_factory_find(preprocessOverride.constData()), - }; + + m_gstPreprocess = QGstBin::createFromPipelineDescription(preprocessOverride, nullptr, + /*ghostUnlinkedPads=*/true); + if (!m_gstPreprocess) + qCWarning(qLcGstVideoSink) << "Cannot create conversion element:" << preprocessOverride; } - if (!factory) - factory = QGstElementFactoryHandle{ - gst_element_factory_find("imxvideoconvert_g2d"), + if (!m_gstPreprocess) { + // This is a hack for some iMX and NVidia platforms. These require the use of a special + // video conversion element in the pipeline before the video sink, as they unfortunately + // output some proprietary format from the decoder even though it's sometimes marked as + // a regular supported video/x-raw format. + static constexpr auto decodersToTest = { + "imxvideoconvert_g2d", + "nvvidconv", }; - if (!factory) - factory = QGstElementFactoryHandle{ - gst_element_factory_find("nvvidconv"), - }; + for (const char *decoder : decodersToTest) { + factory = QGstElement::findFactory(decoder); + if (factory) + break; + } - if (factory) { - qCDebug(qLcGstVideoSink) << "instantiating conversion element: " - << g_type_name( - gst_element_factory_get_element_type(factory.get())); + if (factory) { + qCDebug(qLcGstVideoSink) + << "instantiating conversion element:" + << g_type_name(gst_element_factory_get_element_type(factory.get())); - gstPreprocess = QGstElement::createFromFactory(factory, "preprocess"); + m_gstPreprocess = QGstElement::createFromFactory(factory, "preprocess"); + } } bool disablePixelAspectRatio = @@ -91,32 +100,35 @@ QGstreamerVideoSink::QGstreamerVideoSink(QVideoSink *parent) // pixel-aspect-ratio handling // // compare: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6242 - gstCapsFilter = + m_gstCapsFilter = QGstElement::createFromFactory("identity", "nullPixelAspectRatioCapsFilter"); } else { - gstCapsFilter = QGstElement::createFromFactory("capsfilter", "pixelAspectRatioCapsFilter"); + m_gstCapsFilter = + QGstElement::createFromFactory("capsfilter", "pixelAspectRatioCapsFilter"); QGstCaps capsFilterCaps{ gst_caps_new_simple("video/x-raw", "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL), QGstCaps::HasRef, }; - g_object_set(gstCapsFilter.element(), "caps", capsFilterCaps.caps(), NULL); + g_object_set(m_gstCapsFilter.element(), "caps", capsFilterCaps.caps(), NULL); } - if (gstPreprocess) { - sinkBin.add(gstQueue, gstPreprocess, gstCapsFilter); - qLinkGstElements(gstQueue, gstPreprocess, gstCapsFilter); + if (m_gstPreprocess) { + m_sinkBin.add(m_gstPreprocess, m_gstCapsFilter); + qLinkGstElements(m_gstPreprocess, m_gstCapsFilter); + m_sinkBin.addGhostPad(m_gstPreprocess, "sink"); } else { - sinkBin.add(gstQueue, gstCapsFilter); - qLinkGstElements(gstQueue, gstCapsFilter); + m_sinkBin.add(m_gstCapsFilter); + m_sinkBin.addGhostPad(m_gstCapsFilter, "sink"); } - sinkBin.addGhostPad(gstQueue, "sink"); - gstSubtitleSink = + m_gstSubtitleSink = QGstElement(GST_ELEMENT(QGstSubtitleSink::createSink(this)), QGstElement::NeedsRef); } QGstreamerVideoSink::~QGstreamerVideoSink() { + emit aboutToBeDestroyed(); + unrefGstContexts(); setPipeline(QGstPipeline()); @@ -125,19 +137,19 @@ QGstreamerVideoSink::~QGstreamerVideoSink() QGstElement QGstreamerVideoSink::gstSink() { updateSinkElement(); - return sinkBin; + return m_sinkBin; } void QGstreamerVideoSink::setPipeline(QGstPipeline pipeline) { - gstPipeline = std::move(pipeline); + m_pipeline = std::move(pipeline); } bool QGstreamerVideoSink::inStoppedState() const { - if (gstPipeline.isNull()) + if (m_pipeline.isNull()) return true; - return gstPipeline.inStoppedState(); + return m_pipeline.inStoppedState(); } void QGstreamerVideoSink::setRhi(QRhi *rhi) @@ -149,7 +161,7 @@ void QGstreamerVideoSink::setRhi(QRhi *rhi) m_rhi = rhi; updateGstContexts(); - if (!gstQtSink.isNull()) { + if (!m_gstQtSink.isNull()) { // force creation of a new sink with proper caps createQtSink(); updateSinkElement(); @@ -158,36 +170,37 @@ void QGstreamerVideoSink::setRhi(QRhi *rhi) void QGstreamerVideoSink::createQtSink() { - if (gstQtSink) - gstQtSink.setStateSync(GST_STATE_NULL); + if (m_gstQtSink) + m_gstQtSink.setStateSync(GST_STATE_NULL); - gstQtSink = QGstElement(reinterpret_cast<GstElement *>(QGstVideoRendererSink::createSink(this)), - QGstElement::NeedsRef); + m_gstQtSink = + QGstElement(reinterpret_cast<GstElement *>(QGstVideoRendererSink::createSink(this)), + QGstElement::NeedsRef); } void QGstreamerVideoSink::updateSinkElement() { QGstElement newSink; - if (gstQtSink.isNull()) + if (m_gstQtSink.isNull()) createQtSink(); - newSink = gstQtSink; + newSink = m_gstQtSink; - if (newSink == gstVideoSink) + if (newSink == m_gstVideoSink) return; - gstPipeline.modifyPipelineWhileNotRunning([&] { - if (!gstVideoSink.isNull()) - sinkBin.stopAndRemoveElements(gstVideoSink); + m_pipeline.modifyPipelineWhileNotRunning([&] { + if (!m_gstVideoSink.isNull()) + m_sinkBin.stopAndRemoveElements(m_gstVideoSink); newSink.set("async", false); // no asynchronous state changes - gstVideoSink = newSink; - sinkBin.add(gstVideoSink); - qLinkGstElements(gstCapsFilter, gstVideoSink); - gstVideoSink.setState(GST_STATE_PAUSED); + m_gstVideoSink = newSink; + m_sinkBin.add(m_gstVideoSink); + qLinkGstElements(m_gstCapsFilter, m_gstVideoSink); + m_gstVideoSink.setState(GST_STATE_PAUSED); }); - gstPipeline.dumpGraph("updateVideoSink"); + m_pipeline.dumpGraph("updateVideoSink"); } void QGstreamerVideoSink::unrefGstContexts() @@ -200,6 +213,8 @@ void QGstreamerVideoSink::unrefGstContexts() void QGstreamerVideoSink::updateGstContexts() { + using namespace Qt::Literals; + unrefGstContexts(); #if QT_CONFIG(gstreamer_gl) @@ -212,12 +227,12 @@ void QGstreamerVideoSink::updateGstContexts() const QString platform = QGuiApplication::platformName(); QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface(); - m_eglDisplay = pni->nativeResourceForIntegration("egldisplay"); + m_eglDisplay = pni->nativeResourceForIntegration("egldisplay"_ba); // qDebug() << "platform is" << platform << m_eglDisplay; QGstGLDisplayHandle gstGlDisplay; - const char *contextName = "eglcontext"; + QByteArray contextName = "eglcontext"_ba; GstGLPlatform glPlatform = GST_GL_PLATFORM_EGL; // use the egl display if we have one if (m_eglDisplay) { @@ -227,12 +242,12 @@ void QGstreamerVideoSink::updateGstContexts() m_eglImageTargetTexture2D = eglGetProcAddress("glEGLImageTargetTexture2DOES"); #endif } else { - auto display = pni->nativeResourceForIntegration("display"); + auto display = pni->nativeResourceForIntegration("display"_ba); if (display) { #if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") if (platform == QLatin1String("xcb")) { - contextName = "glxcontext"; + contextName = "glxcontext"_ba; glPlatform = GST_GL_PLATFORM_GLX; gstGlDisplay.reset(GST_GL_DISPLAY_CAST( @@ -289,8 +304,8 @@ void QGstreamerVideoSink::updateGstContexts() gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext.get(), nullptr); displayContext.close(); - if (!gstPipeline.isNull()) - gst_element_set_context(gstPipeline.element(), m_gstGlLocalContext.get()); + if (m_pipeline) + gst_element_set_context(m_pipeline.element(), m_gstGlLocalContext.get()); #endif // #if QT_CONFIG(gstreamer_gl) } diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h index 132eab557..7ee1dd2e6 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h @@ -15,25 +15,17 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qplatformvideosink_p.h> +#include <QtMultimedia/qvideosink.h> +#include <QtMultimedia/private/qplatformvideosink_p.h> #include <common/qgstpipeline_p.h> -#include <common/qgstreamervideooverlay_p.h> -#include <QtGui/qcolor.h> -#include <qvideosink.h> - -#if QT_CONFIG(gstreamer_gl) -#include <gst/gl/gl.h> -#endif QT_BEGIN_NAMESPACE -class QGstreamerVideoRenderer; -class QVideoWindow; class QGstreamerVideoSink : public QPlatformVideoSink { Q_OBJECT + public: explicit QGstreamerVideoSink(QVideoSink *parent = nullptr); ~QGstreamerVideoSink(); @@ -42,7 +34,7 @@ public: QRhi *rhi() const { return m_rhi; } QGstElement gstSink(); - QGstElement subtitleSink() const { return gstSubtitleSink; } + QGstElement subtitleSink() const { return m_gstSubtitleSink; } void setPipeline(QGstPipeline pipeline); bool inStoppedState() const; @@ -52,6 +44,9 @@ public: Qt::HANDLE eglDisplay() const { return m_eglDisplay; } QFunctionPointer eglImageTargetTexture2D() const { return m_eglImageTargetTexture2D; } +Q_SIGNALS: + void aboutToBeDestroyed(); + private: void createQtSink(); void updateSinkElement(); @@ -59,14 +54,13 @@ private: void unrefGstContexts(); void updateGstContexts(); - QGstPipeline gstPipeline; - QGstBin sinkBin; - QGstElement gstQueue; - QGstElement gstPreprocess; - QGstElement gstCapsFilter; - QGstElement gstVideoSink; - QGstElement gstQtSink; - QGstElement gstSubtitleSink; + QGstPipeline m_pipeline; + QGstBin m_sinkBin; + QGstElement m_gstPreprocess; + QGstElement m_gstCapsFilter; + QGstElement m_gstVideoSink; + QGstElement m_gstQtSink; + QGstElement m_gstSubtitleSink; QRhi *m_rhi = nullptr; diff --git a/src/plugins/multimedia/gstreamer/common/qgstutils.cpp b/src/plugins/multimedia/gstreamer/common/qgstutils.cpp index d3d5f6124..8ec2bde3c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstutils.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstutils.cpp @@ -56,7 +56,7 @@ QAudioFormat QGstUtils::audioFormatForSample(GstSample *sample) QAudioFormat QGstUtils::audioFormatForCaps(const QGstCaps &caps) { QAudioFormat format; - QGstStructure s = caps.at(0); + QGstStructureView s = caps.at(0); if (s.name() != "audio/x-raw") return format; @@ -115,7 +115,7 @@ QList<QAudioFormat::SampleFormat> QGValue::getSampleFormats() const return formats; } -void QGstUtils::setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer) +void QGstUtils::setFrameTimeStampsFromBuffer(QVideoFrame *frame, GstBuffer *buffer) { using namespace std::chrono; using namespace std::chrono_literals; diff --git a/src/plugins/multimedia/gstreamer/common/qgstutils_p.h b/src/plugins/multimedia/gstreamer/common/qgstutils_p.h index 4141ae0bf..c65fcf090 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstutils_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstutils_p.h @@ -31,7 +31,7 @@ QAudioFormat audioFormatForSample(GstSample *sample); QAudioFormat audioFormatForCaps(const QGstCaps &caps); QGstCaps capsForAudioFormat(const QAudioFormat &format); -void setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer); +void setFrameTimeStampsFromBuffer(QVideoFrame *frame, GstBuffer *buffer); } // namespace QGstUtils GList *qt_gst_video_sinks(); diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp index 101d56af6..df3fb3d69 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp @@ -51,18 +51,19 @@ QT_BEGIN_NAMESPACE #define DRM_FORMAT_GR1616 fourcc_code('G', 'R', '3', '2') /* [31:0] G:R 16:16 little endian */ #define DRM_FORMAT_BGRA1010102 fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */ -QGstVideoBuffer::QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, - const QVideoFrameFormat &frameFormat, +QGstVideoBuffer::QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, + QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format) - : QAbstractVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) ? - QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, sink ? sink->rhi() : nullptr) - , memoryFormat(format) - , m_frameFormat(frameFormat) - , m_rhi(sink ? sink->rhi() : nullptr) - , m_videoInfo(info) - , m_buffer(buffer) + : QHwVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) + ? QVideoFrame::RhiTextureHandle + : QVideoFrame::NoHandle, + sink ? sink->rhi() : nullptr), + memoryFormat(format), + m_frameFormat(frameFormat), + m_rhi(sink ? sink->rhi() : nullptr), + m_videoInfo(info), + m_buffer(std::move(buffer)) { - gst_buffer_ref(m_buffer); if (sink) { eglDisplay = sink->eglDisplay(); eglImageTargetTexture2D = sink->eglImageTargetTexture2D(); @@ -75,42 +76,35 @@ QGstVideoBuffer::QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QG QGstVideoBuffer::~QGstVideoBuffer() { - unmap(); - - gst_buffer_unref(m_buffer); + Q_ASSERT(m_mode == QtVideo::MapMode::NotMapped); } - -QVideoFrame::MapMode QGstVideoBuffer::mapMode() const +QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QtVideo::MapMode mode) { - return m_mode; -} - -QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) -{ - const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0) - | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0)); + const GstMapFlags flags = GstMapFlags( + ((mode & QtVideo::MapMode::ReadOnly ) == QtVideo::MapMode::NotMapped ? 0 : GST_MAP_READ) + | ((mode & QtVideo::MapMode::WriteOnly) == QtVideo::MapMode::NotMapped ? 0 : GST_MAP_WRITE)); MapData mapData; - if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped) + if (mode == QtVideo::MapMode::NotMapped || m_mode != QtVideo::MapMode::NotMapped) return mapData; if (m_videoInfo.finfo->n_planes == 0) { // Encoded - if (gst_buffer_map(m_buffer, &m_frame.map[0], flags)) { - mapData.nPlanes = 1; + if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) { + mapData.planeCount = 1; mapData.bytesPerLine[0] = -1; - mapData.size[0] = m_frame.map[0].size; + mapData.dataSize[0] = m_frame.map[0].size; mapData.data[0] = static_cast<uchar *>(m_frame.map[0].data); m_mode = mode; } - } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, flags)) { - mapData.nPlanes = GST_VIDEO_FRAME_N_PLANES(&m_frame); + } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer.get(), flags)) { + mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame); for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) { mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i); mapData.data[i] = static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i)); - mapData.size[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); + mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); } m_mode = mode; @@ -120,13 +114,13 @@ QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) void QGstVideoBuffer::unmap() { - if (m_mode != QVideoFrame::NotMapped) { + if (m_mode != QtVideo::MapMode::NotMapped) { if (m_videoInfo.finfo->n_planes == 0) - gst_buffer_unmap(m_buffer, &m_frame.map[0]); + gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]); else gst_video_frame_unmap(&m_frame); } - m_mode = QVideoFrame::NotMapped; + m_mode = QtVideo::MapMode::NotMapped; } #if QT_CONFIG(gstreamer_gl) && QT_CONFIG(linux_dmabuf) @@ -258,9 +252,10 @@ private: std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes]; }; - -static GlTextures mapFromGlTexture(GstBuffer *buffer, GstVideoFrame &frame, GstVideoInfo &videoInfo) +static GlTextures mapFromGlTexture(const QGstBufferHandle &bufferHandle, GstVideoFrame &frame, + GstVideoInfo &videoInfo) { + GstBuffer *buffer = bufferHandle.get(); auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0)); if (!mem) return {}; @@ -293,10 +288,12 @@ static GlTextures mapFromGlTexture(GstBuffer *buffer, GstVideoFrame &frame, GstV } #if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) -static GlTextures mapFromDmaBuffer(QRhi *rhi, GstBuffer *buffer, GstVideoFrame &frame, - GstVideoInfo &videoInfo, Qt::HANDLE eglDisplay, - QFunctionPointer eglImageTargetTexture2D) +static GlTextures mapFromDmaBuffer(QRhi *rhi, const QGstBufferHandle &bufferHandle, + GstVideoFrame &frame, GstVideoInfo &videoInfo, + Qt::HANDLE eglDisplay, QFunctionPointer eglImageTargetTexture2D) { + GstBuffer *buffer = bufferHandle.get(); + Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0))); Q_ASSERT(eglDisplay); Q_ASSERT(eglImageTargetTexture2D); @@ -377,14 +374,15 @@ std::unique_ptr<QVideoFrameTextures> QGstVideoBuffer::mapTextures(QRhi *rhi) #if QT_CONFIG(gstreamer_gl) GlTextures textures = {}; - if (memoryFormat == QGstCaps::GLTexture) { + if (memoryFormat == QGstCaps::GLTexture) textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo); - } -#if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) - else if (memoryFormat == QGstCaps::DMABuf) { - textures = mapFromDmaBuffer(m_rhi, m_buffer, m_frame, m_videoInfo, eglDisplay, eglImageTargetTexture2D); - } -#endif + +# if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) + else if (memoryFormat == QGstCaps::DMABuf) + textures = mapFromDmaBuffer(m_rhi, m_buffer, m_frame, m_videoInfo, eglDisplay, + eglImageTargetTexture2D); + +# endif if (textures.count > 0) return std::make_unique<QGstQVideoFrameTextures>(rhi, QSize{m_videoInfo.width, m_videoInfo.height}, m_frameFormat.pixelFormat(), textures); diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h index 27567c9f7..573a4662c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h @@ -15,8 +15,7 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> #include <QtCore/qvariant.h> #include <common/qgst_p.h> @@ -27,18 +26,14 @@ class QVideoFrameFormat; class QGstreamerVideoSink; class QOpenGLContext; -class QGstVideoBuffer final : public QAbstractVideoBuffer +class QGstVideoBuffer final : public QHwVideoBuffer { public: - - QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, + QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format); ~QGstVideoBuffer(); - GstBuffer *buffer() const { return m_buffer; } - QVideoFrame::MapMode mapMode() const override; - - MapData map(QVideoFrame::MapMode mode) override; + MapData map(QtVideo::MapMode mode) override; void unmap() override; std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *) override; @@ -49,8 +44,8 @@ private: QRhi *m_rhi = nullptr; mutable GstVideoInfo m_videoInfo; mutable GstVideoFrame m_frame{}; - GstBuffer *m_buffer = nullptr; - QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; + const QGstBufferHandle m_buffer; + QtVideo::MapMode m_mode = QtVideo::MapMode::NotMapped; Qt::HANDLE eglDisplay = nullptr; QFunctionPointer eglImageTargetTexture2D = nullptr; }; diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp index 0a2de1228..f9c936ea6 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp @@ -20,6 +20,8 @@ #include <common/qgst_debug_p.h> #include <common/qgstutils_p.h> +#include <private/qvideoframe_p.h> + #include <gst/video/video.h> #include <gst/video/gstvideometa.h> @@ -41,6 +43,13 @@ QT_BEGIN_NAMESPACE QGstVideoRenderer::QGstVideoRenderer(QGstreamerVideoSink *sink) : m_sink(sink), m_surfaceCaps(createSurfaceCaps(sink)) { + QObject::connect( + sink, &QGstreamerVideoSink::aboutToBeDestroyed, this, + [this] { + QMutexLocker locker(&m_sinkMutex); + m_sink = nullptr; + }, + Qt::DirectConnection); } QGstVideoRenderer::~QGstVideoRenderer() = default; @@ -111,91 +120,104 @@ const QGstCaps &QGstVideoRenderer::caps() bool QGstVideoRenderer::start(const QGstCaps& caps) { qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << caps; - QMutexLocker locker(&m_mutex); - - m_frameMirrored = false; - m_frameRotationAngle = QtVideo::Rotation::None; - - if (m_active) { - m_flush = true; - m_stop = true; - } - - m_startCaps = caps; - - /* - Waiting for start() to be invoked in the main thread may block - if gstreamer blocks the main thread until this call is finished. - This situation is rare and usually caused by setState(Null) - while pipeline is being prerolled. - The proper solution to this involves controlling gstreamer pipeline from - other thread than video surface. - - Currently start() fails if wait() timed out. - */ - if (!waitForAsyncEvent(&locker, &m_setupCondition, 1000) && !m_startCaps.isNull()) { - qWarning() << "Failed to start video surface due to main thread blocked."; - m_startCaps = {}; + { + m_frameRotationAngle = QtVideo::Rotation::None; + auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo(); + if (optionalFormatAndVideoInfo) { + std::tie(m_format, m_videoInfo) = std::move(*optionalFormatAndVideoInfo); + } else { + m_format = {}; + m_videoInfo = {}; + } + m_memoryFormat = caps.memoryFormat(); } - return m_active; + return true; } void QGstVideoRenderer::stop() { - QMutexLocker locker(&m_mutex); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::stop"; - if (!m_active) + QMetaObject::invokeMethod(this, [this] { + m_currentState.buffer = {}; + m_sink->setVideoFrame(QVideoFrame{}); return; - - m_flush = true; - m_stop = true; - - m_startCaps = {}; - - waitForAsyncEvent(&locker, &m_setupCondition, 500); + }); } void QGstVideoRenderer::unlock() { - QMutexLocker locker(&m_mutex); - - m_setupCondition.wakeAll(); - m_renderCondition.wakeAll(); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::unlock"; } -bool QGstVideoRenderer::proposeAllocation(GstQuery *query) +bool QGstVideoRenderer::proposeAllocation(GstQuery *) { - Q_UNUSED(query); - QMutexLocker locker(&m_mutex); - return m_active; + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation"; + return true; } -void QGstVideoRenderer::flush() +GstFlowReturn QGstVideoRenderer::render(GstBuffer *buffer) { - QMutexLocker locker(&m_mutex); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::render"; - m_flush = true; - m_renderBuffer = nullptr; - m_renderCondition.wakeAll(); + GstVideoCropMeta *meta = gst_buffer_get_video_crop_meta(buffer); + if (meta) { + QRect vp(meta->x, meta->y, meta->width, meta->height); + if (m_format.viewport() != vp) { + qCDebug(qLcGstVideoRenderer) + << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x" + << meta->width << " | " << meta->x << "x" << meta->y << "]"; + // Update viewport if data is not the same + m_format.setViewport(vp); + } + } - notify(); -} + RenderBufferState state{ + .buffer = QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef }, + .format = m_format, + .memoryFormat = m_memoryFormat, + .mirrored = m_frameMirrored, + .rotationAngle = m_frameRotationAngle, + }; -GstFlowReturn QGstVideoRenderer::render(GstBuffer *buffer) -{ - QMutexLocker locker(&m_mutex); - qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::render"; + qCDebug(qLcGstVideoRenderer) << " sending video frame"; + + QMetaObject::invokeMethod(this, [this, state = std::move(state)]() mutable { + if (state == m_currentState) { + // same buffer received twice + if (!m_sink || !m_sink->inStoppedState()) + return; - m_renderReturn = GST_FLOW_OK; - m_renderBuffer = buffer; + qCDebug(qLcGstVideoRenderer) << " showing empty video frame"; + m_currentVideoFrame = {}; + m_sink->setVideoFrame(m_currentVideoFrame); + m_currentState = {}; + return; + } - waitForAsyncEvent(&locker, &m_renderCondition, 300); + auto videoBuffer = std::make_unique<QGstVideoBuffer>(state.buffer, m_videoInfo, m_sink, + state.format, state.memoryFormat); + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(videoBuffer), state.format); + QGstUtils::setFrameTimeStampsFromBuffer(&frame, state.buffer.get()); + frame.setMirrored(state.mirrored); + frame.setRotation(state.rotationAngle); + m_currentVideoFrame = std::move(frame); + m_currentState = std::move(state); + + if (!m_sink) + return; + + if (m_sink->inStoppedState()) { + qCDebug(qLcGstVideoRenderer) << " showing empty video frame"; + m_currentVideoFrame = {}; + } - m_renderBuffer = nullptr; + m_sink->setVideoFrame(m_currentVideoFrame); + }); - return m_renderReturn; + return GST_FLOW_OK; } bool QGstVideoRenderer::query(GstQuery *query) @@ -208,6 +230,10 @@ bool QGstVideoRenderer::query(GstQuery *query) if (strcmp(type, "gst.gl.local_context") != 0) return false; + QMutexLocker locker(&m_sinkMutex); + if (!m_sink) + return false; + auto *gstGlContext = m_sink->gstGlLocalContext(); if (!gstGlContext) return false; @@ -224,9 +250,22 @@ bool QGstVideoRenderer::query(GstQuery *query) void QGstVideoRenderer::gstEvent(GstEvent *event) { - if (GST_EVENT_TYPE(event) != GST_EVENT_TAG) + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_TAG: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: Tag"; + return gstEventHandleTag(event); + case GST_EVENT_EOS: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: EOS"; + return gstEventHandleEOS(event); + + default: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: unhandled event - " << event; return; + } +} +void QGstVideoRenderer::gstEventHandleTag(GstEvent *event) +{ GstTagList *taglist = nullptr; gst_event_parse_tag(event, &taglist); if (!taglist) @@ -253,152 +292,28 @@ void QGstVideoRenderer::gstEvent(GstEvent *event) rotationAngle = (180 + atoi(value.get() + flipRotateLen)) % 360; } - QMutexLocker locker(&m_mutex); m_frameMirrored = mirrored; switch (rotationAngle) { - case 0: m_frameRotationAngle = QtVideo::Rotation::None; break; - case 90: m_frameRotationAngle = QtVideo::Rotation::Clockwise90; break; - case 180: m_frameRotationAngle = QtVideo::Rotation::Clockwise180; break; - case 270: m_frameRotationAngle = QtVideo::Rotation::Clockwise270; break; - default: m_frameRotationAngle = QtVideo::Rotation::None; + case 0: + m_frameRotationAngle = QtVideo::Rotation::None; + break; + case 90: + m_frameRotationAngle = QtVideo::Rotation::Clockwise90; + break; + case 180: + m_frameRotationAngle = QtVideo::Rotation::Clockwise180; + break; + case 270: + m_frameRotationAngle = QtVideo::Rotation::Clockwise270; + break; + default: + m_frameRotationAngle = QtVideo::Rotation::None; } } -bool QGstVideoRenderer::event(QEvent *event) +void QGstVideoRenderer::gstEventHandleEOS(GstEvent *) { - if (event->type() == QEvent::UpdateRequest) { - QMutexLocker locker(&m_mutex); - - if (m_notified) { - while (handleEvent(&locker)) {} - m_notified = false; - } - return true; - } - - return QObject::event(event); -} - -bool QGstVideoRenderer::handleEvent(QMutexLocker<QMutex> *locker) -{ - if (m_flush) { - m_flush = false; - if (m_active) { - locker->unlock(); - - if (m_sink && !m_flushed) - m_sink->setVideoFrame(QVideoFrame()); - m_flushed = true; - locker->relock(); - } - } else if (m_stop) { - m_stop = false; - - if (m_active) { - m_active = false; - m_flushed = true; - } - } else if (!m_startCaps.isNull()) { - Q_ASSERT(!m_active); - - auto startCaps = m_startCaps; - m_startCaps = {}; - - if (m_sink) { - locker->unlock(); - - m_flushed = true; - auto optionalFormatAndVideoInfo = startCaps.formatAndVideoInfo(); - if (optionalFormatAndVideoInfo) { - std::tie(m_format, m_videoInfo) = std::move(*optionalFormatAndVideoInfo); - } else { - m_format = {}; - m_videoInfo = {}; - } - - memoryFormat = startCaps.memoryFormat(); - - locker->relock(); - m_active = m_format.isValid(); - } else if (m_active) { - m_active = false; - m_flushed = true; - } - - } else if (m_renderBuffer) { - GstBuffer *buffer = m_renderBuffer; - m_renderBuffer = nullptr; - m_renderReturn = GST_FLOW_ERROR; - - qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::handleEvent(renderBuffer)" << m_active << m_sink; - if (m_active && m_sink) { - gst_buffer_ref(buffer); - - locker->unlock(); - - m_flushed = false; - - GstVideoCropMeta *meta = gst_buffer_get_video_crop_meta(buffer); - if (meta) { - QRect vp(meta->x, meta->y, meta->width, meta->height); - if (m_format.viewport() != vp) { - qCDebug(qLcGstVideoRenderer) << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x" << meta->width << " | " << meta->x << "x" << meta->y << "]"; - // Update viewport if data is not the same - m_format.setViewport(vp); - } - } - - if (m_sink->inStoppedState()) { - qCDebug(qLcGstVideoRenderer) << " sending empty video frame"; - m_sink->setVideoFrame(QVideoFrame()); - } else { - QGstVideoBuffer *videoBuffer = new QGstVideoBuffer(buffer, m_videoInfo, m_sink, m_format, memoryFormat); - QVideoFrame frame(videoBuffer, m_format); - QGstUtils::setFrameTimeStamps(&frame, buffer); - frame.setMirrored(m_frameMirrored); - frame.setRotation(m_frameRotationAngle); - - qCDebug(qLcGstVideoRenderer) << " sending video frame"; - m_sink->setVideoFrame(frame); - } - - gst_buffer_unref(buffer); - - locker->relock(); - - m_renderReturn = GST_FLOW_OK; - } - - m_renderCondition.wakeAll(); - } else { - m_setupCondition.wakeAll(); - - return false; - } - return true; -} - -void QGstVideoRenderer::notify() -{ - if (!m_notified) { - m_notified = true; - QCoreApplication::postEvent(this, new QEvent(QEvent::UpdateRequest)); - } -} - -bool QGstVideoRenderer::waitForAsyncEvent( - QMutexLocker<QMutex> *locker, QWaitCondition *condition, unsigned long time) -{ - if (QThread::currentThread() == thread()) { - while (handleEvent(locker)) {} - m_notified = false; - - return true; - } - - notify(); - - return condition->wait(&m_mutex, time); + stop(); } static GstVideoSinkClass *gvrs_sink_parent_class; @@ -412,8 +327,6 @@ QGstVideoRendererSink *QGstVideoRendererSink::createSink(QGstreamerVideoSink *si QGstVideoRendererSink *gstSink = reinterpret_cast<QGstVideoRendererSink *>( g_object_new(QGstVideoRendererSink::get_type(), nullptr)); - g_signal_connect(G_OBJECT(gstSink), "notify::show-preroll-frame", G_CALLBACK(handleShowPrerollChange), gstSink); - return gstSink; } @@ -509,41 +422,9 @@ void QGstVideoRendererSink::finalize(GObject *object) G_OBJECT_CLASS(gvrs_sink_parent_class)->finalize(object); } -void QGstVideoRendererSink::handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d) -{ - Q_UNUSED(o); - Q_UNUSED(p); - QGstVideoRendererSink *sink = reinterpret_cast<QGstVideoRendererSink *>(d); - - gboolean showPrerollFrame = true; // "show-preroll-frame" property is true by default - g_object_get(G_OBJECT(sink), "show-preroll-frame", &showPrerollFrame, nullptr); - - if (!showPrerollFrame) { - GstState state = GST_STATE_VOID_PENDING; - GstClockTime timeout = 10000000; // 10 ms - gst_element_get_state(GST_ELEMENT(sink), &state, nullptr, timeout); - // show-preroll-frame being set to 'false' while in GST_STATE_PAUSED means - // the QMediaPlayer was stopped from the paused state. - // We need to flush the current frame. - if (state == GST_STATE_PAUSED) - sink->renderer->flush(); - } -} - GstStateChangeReturn QGstVideoRendererSink::change_state( GstElement *element, GstStateChange transition) { - QGstVideoRendererSink *sink = reinterpret_cast<QGstVideoRendererSink *>(element); - - gboolean showPrerollFrame = true; // "show-preroll-frame" property is true by default - g_object_get(G_OBJECT(element), "show-preroll-frame", &showPrerollFrame, nullptr); - - // If show-preroll-frame is 'false' when transitioning from GST_STATE_PLAYING to - // GST_STATE_PAUSED, it means the QMediaPlayer was stopped. - // We need to flush the current frame. - if (transition == GST_STATE_CHANGE_PLAYING_TO_PAUSED && !showPrerollFrame) - sink->renderer->flush(); - return GST_ELEMENT_CLASS(gvrs_sink_parent_class)->change_state(element, transition); } diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h index 6a923ed32..d9e3db462 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h @@ -15,7 +15,11 @@ // We mean it. // +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/qvideoframe.h> #include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtCore/qmutex.h> + #include <gst/video/gstvideosink.h> #include <gst/video/video.h> @@ -30,63 +34,61 @@ #include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QVideoSink; class QGstVideoRenderer : public QObject { public: - explicit QGstVideoRenderer(QGstreamerVideoSink *sink); + explicit QGstVideoRenderer(QGstreamerVideoSink *); ~QGstVideoRenderer(); const QGstCaps &caps(); - bool start(const QGstCaps& caps); + bool start(const QGstCaps &); void stop(); void unlock(); - bool proposeAllocation(GstQuery *query); - - void flush(); - - GstFlowReturn render(GstBuffer *buffer); - - bool event(QEvent *event) override; - bool query(GstQuery *query); - void gstEvent(GstEvent *event); - -private slots: - bool handleEvent(QMutexLocker<QMutex> *locker); + bool proposeAllocation(GstQuery *); + GstFlowReturn render(GstBuffer *); + bool query(GstQuery *); + void gstEvent(GstEvent *); private: void notify(); - bool waitForAsyncEvent(QMutexLocker<QMutex> *locker, QWaitCondition *condition, unsigned long time); static QGstCaps createSurfaceCaps(QGstreamerVideoSink *); - QPointer<QGstreamerVideoSink> m_sink; - - QMutex m_mutex; - QWaitCondition m_setupCondition; - QWaitCondition m_renderCondition; + void gstEventHandleTag(GstEvent *); + void gstEventHandleEOS(GstEvent *); - // --- accessed from multiple threads, need to hold mutex to access - GstFlowReturn m_renderReturn = GST_FLOW_OK; - bool m_active = false; + QMutex m_sinkMutex; + QGstreamerVideoSink *m_sink = nullptr; // written only from qt thread. so only readers on + // worker threads need to acquire the lock + // --- only accessed from gstreamer thread const QGstCaps m_surfaceCaps; - - QGstCaps m_startCaps; - GstBuffer *m_renderBuffer = nullptr; - - bool m_notified = false; - bool m_stop = false; - bool m_flush = false; + QVideoFrameFormat m_format; + GstVideoInfo m_videoInfo{}; + QGstCaps::MemoryFormat m_memoryFormat = QGstCaps::CpuMemory; bool m_frameMirrored = false; QtVideo::Rotation m_frameRotationAngle = QtVideo::Rotation::None; - // --- only accessed from one thread - QVideoFrameFormat m_format; - GstVideoInfo m_videoInfo{}; - bool m_flushed = true; - QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; + // --- only accessed from qt thread + QVideoFrame m_currentVideoFrame; + + struct RenderBufferState + { + QGstBufferHandle buffer; + QVideoFrameFormat format; + QGstCaps::MemoryFormat memoryFormat; + bool mirrored; + QtVideo::Rotation rotationAngle; + + bool operator==(const RenderBufferState &rhs) const + { + return std::tie(buffer, format, memoryFormat, mirrored, rotationAngle) + == std::tie(rhs.buffer, rhs.format, rhs.memoryFormat, rhs.mirrored, + rhs.rotationAngle); + } + }; + RenderBufferState m_currentState; }; class QGstVideoRendererSink @@ -105,8 +107,6 @@ private: static void finalize(GObject *object); - static void handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d); - static GstStateChangeReturn change_state(GstElement *element, GstStateChange transition); static GstCaps *get_caps(GstBaseSink *sink, GstCaps *filter); |