// Copyright (C) 2020 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only //#define DEBUG_DECODER #include "qgstreameraudiodecoder_p.h" #include "qgstreamermessage_p.h" #include #include #include #include #include #include #include #include #include #include #include #define MAX_BUFFERS_IN_QUEUE 4 QT_BEGIN_NAMESPACE typedef enum { GST_PLAY_FLAG_VIDEO = 0x00000001, GST_PLAY_FLAG_AUDIO = 0x00000002, GST_PLAY_FLAG_TEXT = 0x00000004, GST_PLAY_FLAG_VIS = 0x00000008, GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010, GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020, GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040, GST_PLAY_FLAG_DOWNLOAD = 0x00000080, GST_PLAY_FLAG_BUFFERING = 0x000000100 } GstPlayFlags; QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent) : QPlatformAudioDecoder(parent), m_playbin(GST_PIPELINE_CAST(QGstElement("playbin", "playbin").element())) { if (m_playbin.isNull()) { // ### set error return; } // Sort out messages m_playbin.installMessageFilter(this); // Set the rest of the pipeline up setAudioFlags(true); m_audioConvert = QGstElement("audioconvert", "audioconvert"); m_outputBin = QGstBin("audio-output-bin"); m_outputBin.add(m_audioConvert); // add ghostpad m_outputBin.addGhostPad(m_audioConvert, "sink"); g_object_set(m_playbin.object(), "audio-sink", m_outputBin.element(), NULL); g_signal_connect(m_playbin.object(), "deep-notify::source", (GCallback) &QGstreamerAudioDecoder::configureAppSrcElement, (gpointer)this); // Set volume to 100% gdouble volume = 1.0; m_playbin.set("volume", volume); } QGstreamerAudioDecoder::~QGstreamerAudioDecoder() { stop(); #if QT_CONFIG(gstreamer_app) delete m_appSrc; #endif } #if QT_CONFIG(gstreamer_app) void QGstreamerAudioDecoder::configureAppSrcElement(GObject* object, GObject *orig, GParamSpec *pspec, QGstreamerAudioDecoder *self) { Q_UNUSED(object); Q_UNUSED(pspec); // In case we switch from appsrc to not if (!self->appsrc()) return; GstElement *appsrc; g_object_get(orig, "source", &appsrc, NULL); auto *qAppSrc = self->appsrc(); qAppSrc->setExternalAppSrc(appsrc); qAppSrc->setup(self->mDevice); g_object_unref(G_OBJECT(appsrc)); } #endif bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message) { GstMessage* gm = message.rawMessage(); if (gm) { if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_DURATION) { updateDuration(); } else if (GST_MESSAGE_SRC(gm) == m_playbin.object()) { switch (GST_MESSAGE_TYPE(gm)) { case GST_MESSAGE_STATE_CHANGED: { GstState oldState; GstState newState; GstState pending; gst_message_parse_state_changed(gm, &oldState, &newState, &pending); #ifdef DEBUG_DECODER QStringList states; states << "GST_STATE_VOID_PENDING" << "GST_STATE_NULL" << "GST_STATE_READY" << "GST_STATE_PAUSED" << "GST_STATE_PLAYING"; qDebug() << QString("state changed: old: %1 new: %2 pending: %3") \ .arg(states[oldState]) \ .arg(states[newState]) \ .arg(states[pending]) << "internal" << m_state; #endif bool isDecoding = false; switch (newState) { case GST_STATE_VOID_PENDING: case GST_STATE_NULL: case GST_STATE_READY: break; case GST_STATE_PLAYING: isDecoding = true; break; case GST_STATE_PAUSED: isDecoding = true; //gstreamer doesn't give a reliable indication the duration //information is ready, GST_MESSAGE_DURATION is not sent by most elements //the duration is queried up to 5 times with increasing delay m_durationQueries = 5; updateDuration(); break; } setIsDecoding(isDecoding); } break; case GST_MESSAGE_EOS: m_playbin.setState(GST_STATE_NULL); finished(); break; case GST_MESSAGE_ERROR: { GError *err; gchar *debug; gst_message_parse_error(gm, &err, &debug); if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) processInvalidMedia(QAudioDecoder::FormatError, tr("Cannot play stream of type: ")); else processInvalidMedia(QAudioDecoder::ResourceError, QString::fromUtf8(err->message)); qWarning() << "Error:" << QString::fromUtf8(err->message); g_error_free(err); g_free(debug); } break; case GST_MESSAGE_WARNING: { GError *err; gchar *debug; gst_message_parse_warning (gm, &err, &debug); qWarning() << "Warning:" << QString::fromUtf8(err->message); g_error_free (err); g_free (debug); } break; #ifdef DEBUG_DECODER case GST_MESSAGE_INFO: { GError *err; gchar *debug; gst_message_parse_info (gm, &err, &debug); qDebug() << "Info:" << QString::fromUtf8(err->message); g_error_free (err); g_free (debug); } break; #endif default: break; } } else if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_ERROR) { GError *err; gchar *debug; gst_message_parse_error(gm, &err, &debug); QAudioDecoder::Error qerror = QAudioDecoder::ResourceError; if (err->domain == GST_STREAM_ERROR) { switch (err->code) { case GST_STREAM_ERROR_DECRYPT: case GST_STREAM_ERROR_DECRYPT_NOKEY: qerror = QAudioDecoder::AccessDeniedError; break; case GST_STREAM_ERROR_FORMAT: case GST_STREAM_ERROR_DEMUX: case GST_STREAM_ERROR_DECODE: case GST_STREAM_ERROR_WRONG_TYPE: case GST_STREAM_ERROR_TYPE_NOT_FOUND: case GST_STREAM_ERROR_CODEC_NOT_FOUND: qerror = QAudioDecoder::FormatError; break; default: break; } } else if (err->domain == GST_CORE_ERROR) { switch (err->code) { case GST_CORE_ERROR_MISSING_PLUGIN: qerror = QAudioDecoder::FormatError; break; default: break; } } processInvalidMedia(qerror, QString::fromUtf8(err->message)); g_error_free(err); g_free(debug); } } return false; } QUrl QGstreamerAudioDecoder::source() const { return mSource; } void QGstreamerAudioDecoder::setSource(const QUrl &fileName) { stop(); mDevice = nullptr; delete m_appSrc; m_appSrc = nullptr; bool isSignalRequired = (mSource != fileName); mSource = fileName; if (isSignalRequired) emit sourceChanged(); } QIODevice *QGstreamerAudioDecoder::sourceDevice() const { return mDevice; } void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device) { stop(); mSource.clear(); bool isSignalRequired = (mDevice != device); mDevice = device; if (isSignalRequired) emit sourceChanged(); } void QGstreamerAudioDecoder::start() { addAppSink(); if (!mSource.isEmpty()) { m_playbin.set("uri", mSource.toEncoded().constData()); } else if (mDevice) { // make sure we can read from device if (!mDevice->isOpen() || !mDevice->isReadable()) { processInvalidMedia(QAudioDecoder::ResourceError, QLatin1String("Unable to read from specified device")); return; } if (!m_appSrc) m_appSrc = new QGstAppSrc(this); m_playbin.set("uri", "appsrc://"); } else { return; } // Set audio format if (m_appSink) { if (mFormat.isValid()) { setAudioFlags(false); auto caps = QGstUtils::capsForAudioFormat(mFormat); gst_app_sink_set_caps(m_appSink, caps.get()); } else { // We want whatever the native audio format is setAudioFlags(true); gst_app_sink_set_caps(m_appSink, nullptr); } } if (m_playbin.setState(GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { qWarning() << "GStreamer; Unable to start decoding process"; m_playbin.dumpGraph("failed"); return; } } void QGstreamerAudioDecoder::stop() { m_playbin.setState(GST_STATE_NULL); removeAppSink(); // GStreamer thread is stopped. Can safely access m_buffersAvailable if (m_buffersAvailable != 0) { m_buffersAvailable = 0; emit bufferAvailableChanged(false); } if (m_position != -1) { m_position = -1; emit positionChanged(m_position); } if (m_duration != -1) { m_duration = -1; emit durationChanged(m_duration); } setIsDecoding(false); } QAudioFormat QGstreamerAudioDecoder::audioFormat() const { return mFormat; } void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format) { if (mFormat != format) { mFormat = format; emit formatChanged(mFormat); } } QAudioBuffer QGstreamerAudioDecoder::read() { QAudioBuffer audioBuffer; int buffersAvailable; { QMutexLocker locker(&m_buffersMutex); buffersAvailable = m_buffersAvailable; // need to decrement before pulling a buffer // to make sure assert in QGstreamerAudioDecoderControl::new_buffer works m_buffersAvailable--; } if (buffersAvailable) { if (buffersAvailable == 1) emit bufferAvailableChanged(false); const char* bufferData = nullptr; int bufferSize = 0; GstSample *sample = gst_app_sink_pull_sample(m_appSink); GstBuffer *buffer = gst_sample_get_buffer(sample); GstMapInfo mapInfo; gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); bufferData = (const char*)mapInfo.data; bufferSize = mapInfo.size; QAudioFormat format = QGstUtils::audioFormatForSample(sample); if (format.isValid()) { // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. qint64 position = getPositionFromBuffer(buffer); audioBuffer = QAudioBuffer(QByteArray((const char*)bufferData, bufferSize), format, position); position /= 1000; // convert to milliseconds if (position != m_position) { m_position = position; emit positionChanged(m_position); } } gst_buffer_unmap(buffer, &mapInfo); gst_sample_unref(sample); } return audioBuffer; } bool QGstreamerAudioDecoder::bufferAvailable() const { QMutexLocker locker(&m_buffersMutex); return m_buffersAvailable > 0; } qint64 QGstreamerAudioDecoder::position() const { return m_position; } qint64 QGstreamerAudioDecoder::duration() const { return m_duration; } void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString) { stop(); emit error(int(errorCode), errorString); } GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *, gpointer user_data) { // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()." QGstreamerAudioDecoder *decoder = reinterpret_cast(user_data); int buffersAvailable; { QMutexLocker locker(&decoder->m_buffersMutex); buffersAvailable = decoder->m_buffersAvailable; decoder->m_buffersAvailable++; Q_ASSERT(decoder->m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE); } if (!buffersAvailable) decoder->bufferAvailableChanged(true); decoder->bufferReady(); return GST_FLOW_OK; } void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) { int flags = m_playbin.getInt("flags"); // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired // it prevents audio format conversion flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO); flags |= GST_PLAY_FLAG_AUDIO; if (wantNativeAudio) flags |= GST_PLAY_FLAG_NATIVE_AUDIO; m_playbin.set("flags", flags); } void QGstreamerAudioDecoder::addAppSink() { if (m_appSink) return; m_appSink = (GstAppSink*)gst_element_factory_make("appsink", nullptr); GstAppSinkCallbacks callbacks; memset(&callbacks, 0, sizeof(callbacks)); callbacks.new_sample = &new_sample; gst_app_sink_set_callbacks(m_appSink, &callbacks, this, nullptr); gst_app_sink_set_max_buffers(m_appSink, MAX_BUFFERS_IN_QUEUE); gst_base_sink_set_sync(GST_BASE_SINK(m_appSink), FALSE); gst_bin_add(m_outputBin.bin(), GST_ELEMENT(m_appSink)); gst_element_link(m_audioConvert.element(), GST_ELEMENT(m_appSink)); } void QGstreamerAudioDecoder::removeAppSink() { if (!m_appSink) return; gst_element_unlink(m_audioConvert.element(), GST_ELEMENT(m_appSink)); gst_bin_remove(m_outputBin.bin(), GST_ELEMENT(m_appSink)); m_appSink = nullptr; } void QGstreamerAudioDecoder::updateDuration() { int duration = m_playbin.duration() / 1000000; if (m_duration != duration) { m_duration = duration; emit durationChanged(m_duration); } if (m_duration > 0) m_durationQueries = 0; if (m_durationQueries > 0) { //increase delay between duration requests int delay = 25 << (5 - m_durationQueries); QTimer::singleShot(delay, this, SLOT(updateDuration())); m_durationQueries--; } } qint64 QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer* buffer) { qint64 position = GST_BUFFER_TIMESTAMP(buffer); if (position >= 0) position = position / G_GINT64_CONSTANT(1000); // microseconds else position = -1; return position; } QT_END_NAMESPACE #include "moc_qgstreameraudiodecoder_p.cpp"