From 6468334eb922af18f02ea196d2ac0aff00c71a74 Mon Sep 17 00:00:00 2001 From: Lev Zelenskiy Date: Thu, 16 Feb 2012 16:54:46 +1000 Subject: GStreamer backend for audio decoder service. Includes basic integration test. Change-Id: I4c6d1dbefa1f27e107b3556a3d4da58811eeb122 Reviewed-by: Michael Goddard --- .../audiodecoder/qgstreameraudiodecodercontrol.cpp | 1 + .../audiodecoder/qgstreameraudiodecodersession.cpp | 239 +++++++++++++++------ .../audiodecoder/qgstreameraudiodecodersession.h | 15 +- 3 files changed, 182 insertions(+), 73 deletions(-) (limited to 'src/plugins/gstreamer/audiodecoder') diff --git a/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodercontrol.cpp b/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodercontrol.cpp index db1b716eb..4668a39bb 100644 --- a/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodercontrol.cpp +++ b/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodercontrol.cpp @@ -62,6 +62,7 @@ QGstreamerAudioDecoderControl::QGstreamerAudioDecoderControl(QGstreamerAudioDeco connect(m_session, SIGNAL(bufferReady()), this, SIGNAL(bufferReady())); connect(m_session, SIGNAL(error(int,QString)), this, SIGNAL(error(int,QString))); connect(m_session, SIGNAL(formatChanged(QAudioFormat)), this, SIGNAL(formatChanged(QAudioFormat))); + connect(m_session, SIGNAL(sourceChanged()), this, SIGNAL(sourceChanged())); connect(m_session, SIGNAL(stateChanged(QAudioDecoder::State)), this, SIGNAL(stateChanged(QAudioDecoder::State))); } diff --git a/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.cpp b/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.cpp index 4afee231d..88f3a0e24 100644 --- a/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.cpp +++ b/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.cpp @@ -54,9 +54,24 @@ #include #include #include +#include + +#define MAX_BUFFERS_IN_QUEUE 5 QT_BEGIN_NAMESPACE +typedef enum { + GST_PLAY_FLAG_VIDEO = 0x00000001, + GST_PLAY_FLAG_AUDIO = 0x00000002, + GST_PLAY_FLAG_TEXT = 0x00000004, + GST_PLAY_FLAG_VIS = 0x00000008, + GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010, + GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020, + GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040, + GST_PLAY_FLAG_DOWNLOAD = 0x00000080, + GST_PLAY_FLAG_BUFFERING = 0x000000100 +} GstPlayFlags; + QGstreamerAudioDecoderSession::QGstreamerAudioDecoderSession(QObject *parent) : QObject(parent), m_state(QAudioDecoder::StoppedState), @@ -64,28 +79,67 @@ QGstreamerAudioDecoderSession::QGstreamerAudioDecoderSession(QObject *parent) m_busHelper(0), m_bus(0), m_playbin(0), + m_appSink(0), #if defined(HAVE_GST_APPSRC) m_appSrc(0), #endif - mDevice(0) + mDevice(0), + m_buffersAvailable(0) { // Default format mFormat.setChannels(2); mFormat.setSampleSize(16); mFormat.setFrequency(48000); - mFormat.setCodec("audio/x-raw"); + mFormat.setCodec("audio/pcm"); mFormat.setSampleType(QAudioFormat::UnSignedInt); // Create pipeline here -#if 0 + m_playbin = gst_element_factory_make("playbin2", NULL); + if (m_playbin != 0) { + + int flags = 0; + g_object_get(G_OBJECT(m_playbin), "flags", &flags, NULL); + // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO, it prevents audio format conversion + flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO); + flags |= GST_PLAY_FLAG_AUDIO; + g_object_set(G_OBJECT(m_playbin), "flags", flags, NULL); + + m_appSink = (GstAppSink*)gst_element_factory_make("appsink", NULL); + gst_object_ref(GST_OBJECT(m_appSink)); + + GstAppSinkCallbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.new_buffer = &new_buffer; + gst_app_sink_set_callbacks(m_appSink, &callbacks, this, NULL); + gst_app_sink_set_max_buffers(m_appSink, MAX_BUFFERS_IN_QUEUE); + gst_base_sink_set_sync(GST_BASE_SINK(m_appSink), FALSE); + + GstElement *audioConvert = gst_element_factory_make("audioconvert", NULL); + + GstElement *bin = gst_bin_new("audio-output-bin"); + gst_bin_add(GST_BIN(bin), audioConvert); + gst_bin_add(GST_BIN(bin), GST_ELEMENT(m_appSink)); + gst_element_link(audioConvert, GST_ELEMENT(m_appSink)); + + // add ghostpad + GstPad *pad = gst_element_get_static_pad(audioConvert, "sink"); + Q_ASSERT(pad); + gst_element_add_pad(GST_ELEMENT(bin), gst_ghost_pad_new("sink", pad)); + gst_object_unref(GST_OBJECT(pad)); + // Sort out messages m_bus = gst_element_get_bus(m_playbin); m_busHelper = new QGstreamerBusHelper(m_bus, this); m_busHelper->installMessageFilter(this); + + g_object_set(G_OBJECT(m_playbin), "audio-sink", bin, NULL); + + // Set volume to 100% + gdouble volume = 1.0; + g_object_set(G_OBJECT(m_playbin), "volume", volume, NULL); } -#endif } QGstreamerAudioDecoderSession::~QGstreamerAudioDecoderSession() @@ -96,6 +150,7 @@ QGstreamerAudioDecoderSession::~QGstreamerAudioDecoderSession() delete m_busHelper; gst_object_unref(GST_OBJECT(m_bus)); gst_object_unref(GST_OBJECT(m_playbin)); + gst_object_unref(GST_OBJECT(m_appSink)); } } @@ -116,66 +171,6 @@ void QGstreamerAudioDecoderSession::configureAppSrcElement(GObject* object, GObj } #endif -#if 0 -void QGstreamerAudioDecoderSession::loadFromStream(const QNetworkRequest &request, QIODevice *appSrcStream) -{ -#if defined(HAVE_GST_APPSRC) -#ifdef DEBUG_PLAYBIN - qDebug() << Q_FUNC_INFO; -#endif - m_request = request; - m_duration = -1; - m_lastPosition = 0; - m_haveQueueElement = false; - - if (m_appSrc) - m_appSrc->deleteLater(); - m_appSrc = new QGstAppSrc(this); - m_appSrc->setStream(appSrcStream); - - if (m_playbin) { - m_tags.clear(); - emit tagsChanged(); - - g_signal_connect(G_OBJECT(m_playbin), "deep-notify::source", (GCallback) &QGstreamerAudioDecoderSession::configureAppSrcElement, (gpointer)this); - g_object_set(G_OBJECT(m_playbin), "uri", "appsrc://", NULL); - - if (!m_streamTypes.isEmpty()) { - m_streamProperties.clear(); - m_streamTypes.clear(); - - emit streamsChanged(); - } - } -#endif -} - -void QGstreamerAudioDecoderSession::loadFromUri(const QNetworkRequest &request) -{ -#ifdef DEBUG_PLAYBIN - qDebug() << Q_FUNC_INFO << request.url(); -#endif - m_request = request; - m_duration = -1; - m_lastPosition = 0; - m_haveQueueElement = false; - - if (m_playbin) { - m_tags.clear(); - emit tagsChanged(); - - g_object_set(G_OBJECT(m_playbin), "uri", m_request.url().toEncoded().constData(), NULL); - - if (!m_streamTypes.isEmpty()) { - m_streamProperties.clear(); - m_streamTypes.clear(); - - emit streamsChanged(); - } - } -} -#endif - bool QGstreamerAudioDecoderSession::processBusMessage(const QGstreamerMessage &message) { GstMessage* gm = message.rawMessage(); @@ -320,7 +315,10 @@ void QGstreamerAudioDecoderSession::setSourceFilename(const QString &fileName) { stop(); mDevice = 0; + bool isSignalRequired = (mSource != fileName); mSource = fileName; + if (isSignalRequired) + emit sourceChanged(); } QIODevice *QGstreamerAudioDecoderSession::sourceDevice() const @@ -332,17 +330,71 @@ void QGstreamerAudioDecoderSession::setSourceDevice(QIODevice *device) { stop(); mSource.clear(); + bool isSignalRequired = (mDevice != device); mDevice = device; + if (isSignalRequired) + emit sourceChanged(); } void QGstreamerAudioDecoderSession::start() { - // TODO + if (!m_playbin) { + processInvalidMedia(QAudioDecoder::ResourceError, "Playbin element is not valid"); + return; + } + + if (!mSource.isEmpty()) { + g_object_set(G_OBJECT(m_playbin), "uri", QUrl::fromLocalFile(mSource).toEncoded().constData(), NULL); + } else if (mDevice) { + // make sure we can read from device + if (!mDevice->isOpen() || !mDevice->isReadable()) { + processInvalidMedia(QAudioDecoder::AccessDeniedError, "Unable to read from specified device"); + return; + } + + if (m_appSrc) + m_appSrc->deleteLater(); + m_appSrc = new QGstAppSrc(this); + m_appSrc->setStream(mDevice); + + g_signal_connect(G_OBJECT(m_playbin), "deep-notify::source", (GCallback) &QGstreamerAudioDecoderSession::configureAppSrcElement, (gpointer)this); + g_object_set(G_OBJECT(m_playbin), "uri", "appsrc://", NULL); + } else { + return; + } + + // Set audio format + if (m_appSink) { + GstCaps *caps = QGstUtils::capsForAudioFormat(mFormat); + gst_app_sink_set_caps(m_appSink, caps); // appsink unrefs caps + } + + m_pendingState = QAudioDecoder::DecodingState; + if (gst_element_set_state(m_playbin, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + qWarning() << "GStreamer; Unable to start decoding process"; + m_pendingState = m_state = QAudioDecoder::StoppedState; + + emit stateChanged(m_state); + } } void QGstreamerAudioDecoderSession::stop() { - // TODO + if (m_playbin) { + gst_element_set_state(m_playbin, GST_STATE_NULL); + + QAudioDecoder::State oldState = m_state; + m_pendingState = m_state = QAudioDecoder::StoppedState; + + // GStreamer thread is stopped. Can safely access m_buffersAvailable + if (m_buffersAvailable != 0) { + m_buffersAvailable = 0; + emit bufferAvailableChanged(false); + } + + if (oldState != m_state) + emit stateChanged(m_state); + } } QAudioFormat QGstreamerAudioDecoderSession::audioFormat() const @@ -360,15 +412,43 @@ void QGstreamerAudioDecoderSession::setAudioFormat(const QAudioFormat &format) QAudioBuffer QGstreamerAudioDecoderSession::read(bool *ok) { - // TODO + QAudioBuffer audioBuffer; + + int buffersAvailable; + { + QMutexLocker locker(&m_buffersMutex); + buffersAvailable = m_buffersAvailable; + + // need to decrement before pulling a buffer + // to make sure assert in QGstreamerAudioDecoderSession::new_buffer works + m_buffersAvailable--; + } + + + if (buffersAvailable) { + if (buffersAvailable == 1) + emit bufferAvailableChanged(false); + + GstBuffer *buffer = gst_app_sink_pull_buffer(m_appSink); + + QAudioFormat format = QGstUtils::audioFormatForBuffer(buffer); + if (format.isValid()) { + // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. + // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. + audioBuffer = QAudioBuffer(QByteArray((const char*)buffer->data, buffer->size), format); + } + gst_buffer_unref(buffer); + } + if (ok) - *ok = false; - return QAudioBuffer(); + *ok = audioBuffer.isValid(); + return audioBuffer; } bool QGstreamerAudioDecoderSession::bufferAvailable() const { - return false; + QMutexLocker locker(&m_buffersMutex); + return m_buffersAvailable; } void QGstreamerAudioDecoderSession::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString) @@ -377,4 +457,23 @@ void QGstreamerAudioDecoderSession::processInvalidMedia(QAudioDecoder::Error err emit error(int(errorCode), errorString); } +GstFlowReturn QGstreamerAudioDecoderSession::new_buffer(GstAppSink *, gpointer user_data) +{ + // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()." + QGstreamerAudioDecoderSession *session = reinterpret_cast(user_data); + + int buffersAvailable; + { + QMutexLocker locker(&session->m_buffersMutex); + buffersAvailable = session->m_buffersAvailable; + session->m_buffersAvailable++; + Q_ASSERT(session->m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE); + } + + if (!buffersAvailable) + QMetaObject::invokeMethod(session, "bufferAvailableChanged", Qt::QueuedConnection, Q_ARG(bool, true)); + QMetaObject::invokeMethod(session, "bufferReady", Qt::QueuedConnection); + return GST_FLOW_OK; +} + QT_END_NAMESPACE diff --git a/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.h b/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.h index b6ac7516e..df8c8e8dd 100644 --- a/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.h +++ b/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.h @@ -43,6 +43,7 @@ #define QGSTREAMERPLAYERSESSION_H #include +#include #include "qgstreameraudiodecodercontrol.h" #include #include @@ -52,6 +53,7 @@ #endif #include +#include QT_BEGIN_NAMESPACE @@ -95,9 +97,12 @@ public: QAudioBuffer read(bool *ok); bool bufferAvailable() const; + static GstFlowReturn new_buffer(GstAppSink *sink, gpointer user_data); + signals: void stateChanged(QAudioDecoder::State newState); void formatChanged(const QAudioFormat &format); + void sourceChanged(); void error(int error, const QString &errorString); @@ -110,9 +115,10 @@ private: QAudioDecoder::State m_state; QAudioDecoder::State m_pendingState; - QGstreamerBusHelper* m_busHelper; - GstBus* m_bus; - GstElement* m_playbin; + QGstreamerBusHelper *m_busHelper; + GstBus *m_bus; + GstElement *m_playbin; + GstAppSink *m_appSink; #if defined(HAVE_GST_APPSRC) QGstAppSrc *m_appSrc; @@ -121,6 +127,9 @@ private: QString mSource; QIODevice *mDevice; // QWeakPointer perhaps QAudioFormat mFormat; + + mutable QMutex m_buffersMutex; + int m_buffersAvailable; }; QT_END_NAMESPACE -- cgit v1.2.3