GStreamer backend changes for media probing API.

QGstreamerPlayerSession: Using GStreamer buffer probes
to access media data.

Change-Id: Ibc056283fdedaebba90456cc4e86ab63eae5f5f7
Reviewed-by: Michael Goddard <michael.goddard@nokia.com>

3 files changed, 240 insertions, 3 deletions

diff --git a/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp b/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp
index c08032f66..f660cef7b 100644
--- a/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp
+++ b/src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp
@@ -64,6 +64,8 @@
 #endif
 
 #include "qgstreamerstreamscontrol.h"
+#include "qgstreameraudioprobecontrol.h"
+#include "qgstreamervideoprobecontrol.h"
 
 #include <qmediaplaylistnavigator.h>
 #include <qmediaplaylist.h>
@@ -115,6 +117,24 @@ QMediaControl *QGstreamerPlayerService::requestControl(const char *name)
     if (qstrcmp(name,QMediaStreamsControl_iid) == 0)
         return m_streamsControl;
 
+    if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) {
+        if (m_session) {
+            QGstreamerVideoProbeControl *probe = new QGstreamerVideoProbeControl(this);
+            m_session->addProbe(probe);
+            return probe;
+        }
+        return 0;
+    }
+
+    if (qstrcmp(name,QMediaAudioProbeControl_iid) == 0) {
+        if (m_session) {
+            QGstreamerAudioProbeControl *probe = new QGstreamerAudioProbeControl(this);
+            m_session->addProbe(probe);
+            return probe;
+        }
+        return 0;
+    }
+
     if (!m_videoOutput) {
         if (qstrcmp(name, QVideoRendererControl_iid) == 0)
             m_videoOutput = m_videoRenderer;
@@ -140,6 +160,22 @@ void QGstreamerPlayerService::releaseControl(QMediaControl *control)
         m_videoOutput = 0;
         m_control->setVideoOutput(0);
     }
+
+    QGstreamerVideoProbeControl* videoProbe = qobject_cast<QGstreamerVideoProbeControl*>(control);
+    if (videoProbe) {
+        if (m_session)
+            m_session->removeProbe(videoProbe);
+        delete videoProbe;
+        return;
+    }
+
+    QGstreamerAudioProbeControl* audioProbe = qobject_cast<QGstreamerAudioProbeControl*>(control);
+    if (audioProbe) {
+        if (m_session)
+            m_session->removeProbe(audioProbe);
+        delete audioProbe;
+        return;
+    }
 }
 
 QT_END_NAMESPACE
diff --git a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp
index 93ce494d5..708465d26 100644
--- a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp
+++ b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp
@@ -42,6 +42,8 @@
 #include "qgstreamerplayersession.h"
 #include <private/qgstreamerbushelper_p.h>
 
+#include "qgstreameraudioprobecontrol.h"
+#include "qgstreamervideoprobecontrol.h"
 #include "qgstreamervideorendererinterface.h"
 #include "gstvideoconnector.h"
 #include <private/qgstutils_p.h>
@@ -89,6 +91,7 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
      m_videoSink(0),
      m_pendingVideoSink(0),
      m_nullVideoSink(0),
+     m_audioSink(0),
      m_bus(0),
      m_videoOutput(0),
      m_renderer(0),
@@ -96,6 +99,8 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
 #if defined(HAVE_GST_APPSRC)
      m_appSrc(0),
 #endif
+     m_videoBufferProbeId(-1),
+     m_audioBufferProbeId(-1),
      m_volume(100),
      m_playbackRate(1.0),
      m_muted(false),
@@ -128,6 +133,12 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
         flags |= GST_PLAY_FLAG_NATIVE_VIDEO;
 #endif
         g_object_set(G_OBJECT(m_playbin), "flags", flags, NULL);
+
+        m_audioSink = gst_element_factory_make("autoaudiosink", "audiosink");
+        if (m_audioSink) {
+            g_object_set(G_OBJECT(m_playbin), "audio-sink", m_audioSink, NULL);
+            addAudioBufferProbe();
+        }
     } else {
         m_usePlaybin2 = false;
         m_playbin = gst_element_factory_make("playbin", NULL);
@@ -181,6 +192,9 @@ QGstreamerPlayerSession::~QGstreamerPlayerSession()
     if (m_playbin) {
         stop();
 
+        removeVideoBufferProbe();
+        removeAudioBufferProbe();
+
         delete m_busHelper;
         gst_object_unref(GST_OBJECT(m_bus));
         gst_object_unref(GST_OBJECT(m_playbin));
@@ -511,7 +525,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
         qDebug() << "The pipeline has not started yet, pending state:" << m_pendingState;
 #endif
         //the pipeline has not started yet
-        m_pendingVideoSink = 0;        
+        flushVideoProbes();
+        m_pendingVideoSink = 0;
         gst_element_set_state(m_videoSink, GST_STATE_NULL);
         gst_element_set_state(m_playbin, GST_STATE_NULL);
 
@@ -522,6 +537,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
             gst_element_unlink(m_videoIdentity, m_videoSink);
         }
 
+        removeVideoBufferProbe();
+
         gst_bin_remove(GST_BIN(m_videoOutputBin), m_videoSink);
 
         m_videoSink = videoSink;
@@ -544,6 +561,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
             g_object_set(G_OBJECT(m_videoSink), "show-preroll-frame", value, NULL);
         }
 
+        addVideoBufferProbe();
+
         switch (m_pendingState) {
         case QMediaPlayer::PausedState:
             gst_element_set_state(m_playbin, GST_STATE_PAUSED);
@@ -554,6 +573,9 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
         default:
             break;
         }
+
+        resumeVideoProbes();
+
     } else {
         if (m_pendingVideoSink) {
 #ifdef DEBUG_PLAYBIN
@@ -630,6 +652,8 @@ void QGstreamerPlayerSession::finishVideoOutputChange()
         gst_element_unlink(m_videoIdentity, m_videoSink);
     }
 
+    removeVideoBufferProbe();
+
     gst_bin_remove(GST_BIN(m_videoOutputBin), m_videoSink);
 
     m_videoSink = m_pendingVideoSink;
@@ -637,6 +661,8 @@ void QGstreamerPlayerSession::finishVideoOutputChange()
 
     gst_bin_add(GST_BIN(m_videoOutputBin), m_videoSink);
 
+    addVideoBufferProbe();
+
     m_usingColorspaceElement = false;
     bool linked = gst_element_link(m_videoIdentity, m_videoSink);
     if (!linked) {
@@ -680,12 +706,18 @@ void QGstreamerPlayerSession::finishVideoOutputChange()
     if (m_usingColorspaceElement)
         gst_element_set_state(m_colorSpace, state);
 
-    gst_element_set_state(m_videoSink, state);    
+    gst_element_set_state(m_videoSink, state);
+
+    if (state == GST_STATE_NULL)
+        flushVideoProbes();
 
     // Set state change that was deferred due the video output
     // change being pending
     gst_element_set_state(m_playbin, state);
 
+    if (state != GST_STATE_NULL)
+        resumeVideoProbes();
+
     //don't have to wait here, it will unblock eventually
     if (gst_pad_is_blocked(srcPad))
         gst_pad_set_blocked_async(srcPad, false, &block_pad_cb, 0);
@@ -766,8 +798,10 @@ bool QGstreamerPlayerSession::play()
             m_pendingState = m_state = QMediaPlayer::StoppedState;
 
             emit stateChanged(m_state);
-        } else
+        } else {
+            resumeVideoProbes();
             return true;
+        }
     }
 
     return false;
@@ -789,6 +823,7 @@ bool QGstreamerPlayerSession::pause()
 
             emit stateChanged(m_state);
         } else {
+            resumeVideoProbes();
             return true;
         }
     }
@@ -803,9 +838,11 @@ void QGstreamerPlayerSession::stop()
 #endif
     m_everPlayed = false;
     if (m_playbin) {
+
         if (m_renderer)
             m_renderer->stopRenderer();
 
+        flushVideoProbes();
         gst_element_set_state(m_playbin, GST_STATE_NULL);
 
         m_lastPosition = 0;
@@ -1621,4 +1658,140 @@ void QGstreamerPlayerSession::showPrerollFrames(bool enabled)
     }
 }
 
+void QGstreamerPlayerSession::addProbe(QGstreamerVideoProbeControl* probe)
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+
+    if (m_videoProbes.contains(probe))
+        return;
+
+    m_videoProbes.append(probe);
+}
+
+void QGstreamerPlayerSession::removeProbe(QGstreamerVideoProbeControl* probe)
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    m_videoProbes.removeOne(probe);
+    // Do not emit flush signal in this case.
+    // Assume user releases any outstanding references to video frames.
+}
+
+gboolean QGstreamerPlayerSession::padVideoBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data)
+{
+    Q_UNUSED(pad);
+
+    QGstreamerPlayerSession *session = reinterpret_cast<QGstreamerPlayerSession*>(user_data);
+    QMutexLocker locker(&session->m_videoProbeMutex);
+
+    if (session->m_videoProbes.isEmpty())
+        return TRUE;
+
+    foreach (QGstreamerVideoProbeControl* probe, session->m_videoProbes)
+        probe->bufferProbed(buffer);
+
+    return TRUE;
+}
+
+void QGstreamerPlayerSession::addProbe(QGstreamerAudioProbeControl* probe)
+{
+    QMutexLocker locker(&m_audioProbeMutex);
+
+    if (m_audioProbes.contains(probe))
+        return;
+
+    m_audioProbes.append(probe);
+}
+
+void QGstreamerPlayerSession::removeProbe(QGstreamerAudioProbeControl* probe)
+{
+    QMutexLocker locker(&m_audioProbeMutex);
+    m_audioProbes.removeOne(probe);
+}
+
+gboolean QGstreamerPlayerSession::padAudioBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data)
+{
+    Q_UNUSED(pad);
+
+    QGstreamerPlayerSession *session = reinterpret_cast<QGstreamerPlayerSession*>(user_data);
+    QMutexLocker locker(&session->m_audioProbeMutex);
+
+    if (session->m_audioProbes.isEmpty())
+        return TRUE;
+
+    foreach (QGstreamerAudioProbeControl* probe, session->m_audioProbes)
+        probe->bufferProbed(buffer);
+
+    return TRUE;
+}
+
+void QGstreamerPlayerSession::removeVideoBufferProbe()
+{
+    if (m_videoBufferProbeId == -1)
+        return;
+
+    if (!m_videoSink) {
+        m_videoBufferProbeId = -1;
+        return;
+    }
+
+    GstPad *pad = gst_element_get_static_pad(m_videoSink, "sink");
+    if (pad)
+        gst_pad_remove_buffer_probe(pad, m_videoBufferProbeId);
+
+    m_videoBufferProbeId = -1;
+}
+
+void QGstreamerPlayerSession::addVideoBufferProbe()
+{
+    Q_ASSERT(m_videoBufferProbeId == -1);
+    if (!m_videoSink)
+        return;
+
+    GstPad *pad = gst_element_get_static_pad(m_videoSink, "sink");
+    if (pad)
+        m_videoBufferProbeId = gst_pad_add_buffer_probe(pad, G_CALLBACK(padVideoBufferProbe), this);
+}
+
+void QGstreamerPlayerSession::removeAudioBufferProbe()
+{
+    if (m_audioBufferProbeId == -1)
+        return;
+
+    if (!m_audioSink) {
+        m_audioBufferProbeId = -1;
+        return;
+    }
+
+    GstPad *pad = gst_element_get_static_pad(m_audioSink, "sink");
+    if (pad)
+        gst_pad_remove_buffer_probe(pad, m_audioBufferProbeId);
+
+    m_audioBufferProbeId = -1;
+}
+
+void QGstreamerPlayerSession::addAudioBufferProbe()
+{
+    Q_ASSERT(m_audioBufferProbeId == -1);
+    if (!m_audioSink)
+        return;
+
+    GstPad *pad = gst_element_get_static_pad(m_audioSink, "sink");
+    if (pad)
+        m_audioBufferProbeId = gst_pad_add_buffer_probe(pad, G_CALLBACK(padAudioBufferProbe), this);
+}
+
+void QGstreamerPlayerSession::flushVideoProbes()
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    foreach (QGstreamerVideoProbeControl* probe, m_videoProbes)
+        probe->startFlushing();
+}
+
+void QGstreamerPlayerSession::resumeVideoProbes()
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    foreach (QGstreamerVideoProbeControl* probe, m_videoProbes)
+        probe->stopFlushing();
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h
index 8a54e8c01..27b6f347b 100644
--- a/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h
+++ b/src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h
@@ -43,11 +43,13 @@
 #define QGSTREAMERPLAYERSESSION_H
 
 #include <QObject>
+#include <QtCore/qmutex.h>
 #include <QtNetwork/qnetworkrequest.h>
 #include "qgstreamerplayercontrol.h"
 #include <private/qgstreamerbushelper_p.h>
 #include <qmediaplayer.h>
 #include <qmediastreamscontrol.h>
+#include <qaudioformat.h>
 
 #if defined(HAVE_GST_APPSRC)
 #include "qgstappsrc.h"
@@ -61,6 +63,8 @@ class QGstreamerBusHelper;
 class QGstreamerMessage;
 
 class QGstreamerVideoRendererInterface;
+class QGstreamerVideoProbeControl;
+class QGstreamerAudioProbeControl;
 
 class QGstreamerPlayerSession : public QObject,
                                 public QGstreamerBusMessageFilter
@@ -119,6 +123,14 @@ public:
 
     bool isLiveSource() const;
 
+    void addProbe(QGstreamerVideoProbeControl* probe);
+    void removeProbe(QGstreamerVideoProbeControl* probe);
+    static gboolean padVideoBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data);
+
+    void addProbe(QGstreamerAudioProbeControl* probe);
+    void removeProbe(QGstreamerAudioProbeControl* probe);
+    static gboolean padAudioBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data);
+
 public slots:
     void loadFromUri(const QNetworkRequest &url);
     void loadFromStream(const QNetworkRequest &url, QIODevice *stream);
@@ -169,6 +181,13 @@ private:
     static void handleElementAdded(GstBin *bin, GstElement *element, QGstreamerPlayerSession *session);
     void processInvalidMedia(QMediaPlayer::Error errorCode, const QString& errorString);
 
+    void removeVideoBufferProbe();
+    void addVideoBufferProbe();
+    void removeAudioBufferProbe();
+    void addAudioBufferProbe();
+    void flushVideoProbes();
+    void resumeVideoProbes();
+
     QNetworkRequest m_request;
     QMediaPlayer::State m_state;
     QMediaPlayer::State m_pendingState;
@@ -184,6 +203,8 @@ private:
     GstElement* m_pendingVideoSink;
     GstElement* m_nullVideoSink;
 
+    GstElement* m_audioSink;
+
     GstBus* m_bus;
     QObject *m_videoOutput;
     QGstreamerVideoRendererInterface *m_renderer;
@@ -199,6 +220,13 @@ private:
     QList<QMediaStreamsControl::StreamType> m_streamTypes;
     QMap<QMediaStreamsControl::StreamType, int> m_playbin2StreamOffset;
 
+    QList<QGstreamerVideoProbeControl*> m_videoProbes;
+    QMutex m_videoProbeMutex;
+    int m_videoBufferProbeId;
+
+    QList<QGstreamerAudioProbeControl*> m_audioProbes;
+    QMutex m_audioProbeMutex;
+    int m_audioBufferProbeId;
 
     int m_volume;
     qreal m_playbackRate;