Fix QML video rendering

Use the new QVideoSink class to get the video frames.

Fix some APIs, so that we correctly connect VideoOutput and
MediaPlayer.

Change-Id: I65a0d045988c46a917f70dfb922c1bbdb32f6511
Reviewed-by: Doris Verria <doris.verria@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

7 files changed, 41 insertions, 103 deletions

diff --git a/examples/multimedia/video/qmlvideo/qml/qmlvideo/VideoItem.qml b/examples/multimedia/video/qmlvideo/qml/qmlvideo/VideoItem.qml
index 8372acb7e..a98ee2b30 100644
--- a/examples/multimedia/video/qmlvideo/qml/qmlvideo/VideoItem.qml
+++ b/examples/multimedia/video/qmlvideo/qml/qmlvideo/VideoItem.qml
@@ -49,12 +49,11 @@
 ****************************************************************************/
 
 import QtQuick 2.0
-import QtMultimedia 5.0
+import QtMultimedia
 
 VideoOutput {
     id: root
     height: width
-    source: mediaPlayer
 
     property alias duration: mediaPlayer.duration
     property alias mediaSource: mediaPlayer.source
@@ -72,6 +71,7 @@ VideoOutput {
         id: mediaPlayer
         autoLoad: false
         loops: Audio.Infinite
+        videoOutput: root;
 
         onError: {
             if (MediaPlayer.NoError != error) {
diff --git a/src/imports/multimedia/qdeclarativeaudio.cpp b/src/imports/multimedia/qdeclarativeaudio.cpp
index 268d9070b..b84ae0e35 100644
--- a/src/imports/multimedia/qdeclarativeaudio.cpp
+++ b/src/imports/multimedia/qdeclarativeaudio.cpp
@@ -43,6 +43,7 @@
 #include <private/qplatformmediaplayer_p.h>
 
 #include <private/qdeclarativevideooutput_p.h>
+#include <qvideosink.h>
 
 #include "qdeclarativemediametadata_p.h"
 
@@ -140,17 +141,18 @@ void QDeclarativeAudio::setVideoOutput(const QVariant &v)
     if (m_videoOutput == v)
         return;
 
-    QAbstractVideoSurface *surface = nullptr;
+    QVideoSink *sink = nullptr;
     auto vo = v.value<QDeclarativeVideoOutput *>();
     if (vo)
-        surface = vo->videoSurface();
+        sink = vo->videoSink();
     else
-        surface = v.value<QAbstractVideoSurface *>();
+        sink = v.value<QVideoSink *>();
 
     // If only one object has been passed.
-    if (surface) {
-        m_player->setVideoOutput(surface);
+    if (sink) {
+        m_player->setVideoOutput(sink);
     } else {
+#if 0
         QList<QAbstractVideoSurface *> surfaces;
         // Check if it is an array.
         auto arr = v.value<QJSValue>();
@@ -161,7 +163,7 @@ void QDeclarativeAudio::setVideoOutput(const QVariant &v)
                 if (v.isQObject()) {
                     auto obj = v.toQObject();
                     vo = qobject_cast<QDeclarativeVideoOutput *>(obj);
-                    surface = vo ? vo->videoSurface() : qobject_cast<QAbstractVideoSurface *>(obj);
+                    surface = vo ? vo->videoSink() : qobject_cast<QAbstractVideoSurface *>(obj);
                     if (surface)
                         surfaces.append(surface);
                 }
@@ -169,6 +171,7 @@ void QDeclarativeAudio::setVideoOutput(const QVariant &v)
         }
 
         m_player->setVideoOutput(surfaces);
+#endif
     }
 
     m_videoOutput = v;
diff --git a/src/multimedia/video/qvideosink.cpp b/src/multimedia/video/qvideosink.cpp
index 931a4e5ab..c5ed5301c 100644
--- a/src/multimedia/video/qvideosink.cpp
+++ b/src/multimedia/video/qvideosink.cpp
@@ -84,6 +84,7 @@ QVideoSink::QVideoSink(QObject *parent)
     : QObject(parent),
     d(new QVideoSinkPrivate(this))
 {
+    qRegisterMetaType<QVideoFrame>();
 }
 
 QVideoSink::~QVideoSink()
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
index 1cd331279..68d03bb95 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
@@ -46,6 +46,8 @@
 #include <QtMultimedia/qmediacapturesession.h>
 #include <private/qfactoryloader_p.h>
 #include <QtCore/qloggingcategory.h>
+#include <qvideosink.h>
+
 
 static void initResource() {
     Q_INIT_RESOURCE(qtmultimediaquicktools);
@@ -157,9 +159,9 @@ QDeclarativeVideoOutput::~QDeclarativeVideoOutput()
     \sa source
 */
 
-QAbstractVideoSurface *QDeclarativeVideoOutput::videoSurface() const
+QVideoSink *QDeclarativeVideoOutput::videoSink() const
 {
-    return m_backend ? m_backend->videoSurface() : nullptr;
+    return m_backend ? m_backend->videoSink() : nullptr;
 }
 
 /*!
@@ -186,9 +188,9 @@ void QDeclarativeVideoOutput::setSource(QObject *source)
     }
 
     if (QMediaCaptureSession *s = qobject_cast<QMediaCaptureSession *>(source)) {
-        s->setVideoPreview(videoSurface());
+        s->setVideoPreview(videoSink());
     } else if (QMediaPlayer *p = qobject_cast<QMediaPlayer *>(source)) {
-        p->setVideoOutput(videoSurface());
+        p->setVideoOutput(videoSink());
     }
     emit sourceChanged();
 }
@@ -238,12 +240,13 @@ void QDeclarativeVideoOutput::setFillMode(FillMode mode)
     emit fillModeChanged(mode);
 }
 
-void QDeclarativeVideoOutput::_q_updateNativeSize()
+void QDeclarativeVideoOutput::_q_newFrame(const QVideoFrame &frame)
 {
     if (!m_backend)
         return;
 
-    QSize size = m_backend->nativeSize();
+    m_backend->present(frame);
+    QSize size = frame.size();
     if (!qIsDefaultAspect(m_orientation)) {
         size.transpose();
     }
@@ -289,12 +292,8 @@ void QDeclarativeVideoOutput::_q_updateGeometry()
         m_contentRect.moveCenter(rect.center());
     }
 
-    if (m_backend) {
-        if (!m_backend->videoSurface() || m_backend->videoSurface()->isActive())
-            m_backend->updateGeometry();
-        else
-            m_geometryDirty = true;
-    }
+    if (m_backend)
+        m_backend->updateGeometry();
 
 
     if (m_contentRect != oldContentRect)
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h b/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h
index 7a4111602..264744520 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h
@@ -66,7 +66,7 @@ QT_BEGIN_NAMESPACE
 
 class QDeclarativeVideoBackend;
 class QVideoOutputOrientationHandler;
-class QAbstractVideoSurface;
+class QVideoSink;
 
 class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoOutput : public QQuickItem
 {
@@ -80,9 +80,10 @@ class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoOutput : public QQuickItem
     Q_PROPERTY(QRectF contentRect READ contentRect NOTIFY contentRectChanged)
     Q_PROPERTY(QQmlListProperty<QAbstractVideoFilter> filters READ filters);
     Q_PROPERTY(FlushMode flushMode READ flushMode WRITE setFlushMode NOTIFY flushModeChanged REVISION 13)
-    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface CONSTANT REVISION 15)
+    Q_PROPERTY(QVideoSink* videoSink READ videoSink CONSTANT REVISION 15)
     Q_ENUMS(FlushMode)
     Q_ENUMS(FillMode)
+    Q_MOC_INCLUDE(qvideosink.h)
 
 public:
 
@@ -106,7 +107,7 @@ public:
     QObject *source() const { return m_source.data(); }
     void setSource(QObject *source);
 
-    Q_INVOKABLE QAbstractVideoSurface *videoSurface() const;
+    Q_INVOKABLE QVideoSink *videoSink() const;
 
     FillMode fillMode() const;
     void setFillMode(FillMode mode);
@@ -157,7 +158,7 @@ protected:
     void releaseResources() override;
 
 private Q_SLOTS:
-    void _q_updateNativeSize();
+    void _q_newFrame(const QVideoFrame &);
     void _q_updateGeometry();
     void _q_screenOrientationChanged(int);
     void _q_invalidateSceneGraph();
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp
index 10aead543..6ae8aa7be 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp
@@ -45,6 +45,7 @@
 #include <QtCore/qloggingcategory.h>
 #include <private/qmediapluginloader_p.h>
 #include <private/qsgvideonode_p.h>
+#include <qvideosink.h>
 
 #include <QtQuick/QQuickWindow>
 #include <QtCore/QRunnable>
@@ -60,9 +61,10 @@ QDeclarativeVideoBackend::QDeclarativeVideoBackend(QDeclarativeVideoOutput *pare
     : q(parent),
       m_frameChanged(false)
 {
-    m_surface = new QSGVideoItemSurface(this);
-    QObject::connect(m_surface, SIGNAL(surfaceFormatChanged(QVideoSurfaceFormat)),
-                     q, SLOT(_q_updateNativeSize()), Qt::QueuedConnection);
+    m_sink = new QVideoSink(q);
+    qRegisterMetaType<QVideoSurfaceFormat>();
+    QObject::connect(m_sink, SIGNAL(newVideoFrame(const QVideoFrame &)),
+                     q, SLOT(_q_newFrame(const QVideoFrame &)), Qt::QueuedConnection);
 
     // Prioritize the plugin requested by the environment
     QString requestedVideoNode = QString::fromLatin1(qgetenv("QT_VIDEONODE"));
@@ -88,7 +90,7 @@ QDeclarativeVideoBackend::QDeclarativeVideoBackend(QDeclarativeVideoOutput *pare
 
 QDeclarativeVideoBackend::~QDeclarativeVideoBackend()
 {
-    delete m_surface;
+    delete m_sink;
 }
 
 void QDeclarativeVideoBackend::appendFilter(QAbstractVideoFilter *filter)
@@ -286,6 +288,7 @@ QSGNode *QDeclarativeVideoBackend::updatePaintNode(QSGNode *oldNode,
                 nodeFormat.setFrameRate(m_surfaceFormat.frameRate());
                 // Update current surface format if something has changed.
                 m_surfaceFormat = nodeFormat;
+                updateGeometry();
                 videoNode = factory->createNode(nodeFormat);
                 if (videoNode) {
                     qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType()
@@ -326,9 +329,9 @@ QSGNode *QDeclarativeVideoBackend::updatePaintNode(QSGNode *oldNode,
     return videoNode;
 }
 
-QAbstractVideoSurface *QDeclarativeVideoBackend::videoSurface() const
+QVideoSink *QDeclarativeVideoBackend::videoSink() const
 {
-    return m_surface;
+    return m_sink;
 }
 
 QRectF QDeclarativeVideoBackend::adjustedViewport() const
@@ -339,6 +342,7 @@ QRectF QDeclarativeVideoBackend::adjustedViewport() const
 void QDeclarativeVideoBackend::present(const QVideoFrame &frame)
 {
     m_frameMutex.lock();
+    m_surfaceFormat = QVideoSurfaceFormat(frame.size(), frame.pixelFormat(), QVideoFrame::NoHandle);
     m_frame = frame.isValid() ? frame : m_frameOnFlush;
     m_frameChanged = true;
     m_frameMutex.unlock();
@@ -351,58 +355,4 @@ void QDeclarativeVideoBackend::stop()
     present(QVideoFrame());
 }
 
-QSGVideoItemSurface::QSGVideoItemSurface(QDeclarativeVideoBackend *backend, QObject *parent)
-    : QAbstractVideoSurface(parent),
-      m_backend(backend)
-{
-}
-
-QSGVideoItemSurface::~QSGVideoItemSurface() = default;
-
-QList<QVideoFrame::PixelFormat> QSGVideoItemSurface::supportedPixelFormats(
-        QVideoFrame::HandleType handleType) const
-{
-    QList<QVideoFrame::PixelFormat> formats;
-
-    static bool noGLTextures = false;
-    static bool noGLTexturesChecked = false;
-    if (handleType == QVideoFrame::GLTextureHandle) {
-        if (!noGLTexturesChecked) {
-            noGLTexturesChecked = true;
-            noGLTextures = qEnvironmentVariableIsSet("QT_QUICK_NO_TEXTURE_VIDEOFRAMES");
-        }
-        if (noGLTextures)
-            return formats;
-    }
-
-    for (QSGVideoNodeFactoryInterface* factory : qAsConst(m_backend->m_videoNodeFactories))
-        formats.append(factory->supportedPixelFormats(handleType));
-
-    return formats;
-}
-
-bool QSGVideoItemSurface::start(const QVideoSurfaceFormat &format)
-{
-    qCDebug(qLcVideo) << "Video surface format:" << format << "all supported formats:" << supportedPixelFormats(format.handleType());
-    m_backend->m_frameOnFlush = QVideoFrame();
-
-    if (!supportedPixelFormats(format.handleType()).contains(format.pixelFormat()))
-        return false;
-
-    m_backend->m_surfaceFormat = format;
-    return QAbstractVideoSurface::start(format);
-}
-
-void QSGVideoItemSurface::stop()
-{
-    m_backend->stop();
-    QAbstractVideoSurface::stop();
-}
-
-bool QSGVideoItemSurface::present(const QVideoFrame &frame)
-{
-    m_backend->present(frame);
-    return true;
-}
-
 QT_END_NAMESPACE
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h
index 17ed319e0..874b237ca 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h
@@ -63,7 +63,7 @@
 
 QT_BEGIN_NAMESPACE
 
-class QSGVideoItemSurface;
+class QVideoSink;
 class QObject;
 class QAbstractVideoFilter;
 class QVideoFilterRunnable;
@@ -79,7 +79,7 @@ public:
     QSize nativeSize() const;
     void updateGeometry();
     QSGNode *updatePaintNode(QSGNode *oldNode, QQuickItem::UpdatePaintNodeData *data);
-    QAbstractVideoSurface *videoSurface() const;
+    QVideoSink *videoSink() const;
     QRectF adjustedViewport() const;
 
     friend class QSGVideoItemSurface;
@@ -96,7 +96,7 @@ private:
     QDeclarativeVideoOutput *q;
 
     QList<QSGVideoNodeFactoryInterface*> m_videoNodeFactories;
-    QSGVideoItemSurface *m_surface;
+    QVideoSink *m_sink;
     QVideoSurfaceFormat m_surfaceFormat;
 
     QVideoFrame m_frame;
@@ -118,22 +118,6 @@ private:
     QList<Filter> m_filters;
 };
 
-class QSGVideoItemSurface : public QAbstractVideoSurface
-{
-    Q_OBJECT
-public:
-    explicit QSGVideoItemSurface(QDeclarativeVideoBackend *backend, QObject *parent = 0);
-    ~QSGVideoItemSurface();
-    QList<QVideoFrame::PixelFormat> supportedPixelFormats(QVideoFrame::HandleType handleType) const override;
-    bool start(const QVideoSurfaceFormat &format) override;
-    void stop() override;
-    bool present(const QVideoFrame &frame) override;
-
-private:
-    QDeclarativeVideoBackend *m_backend;
-};
-
-
 namespace {
 
 inline bool qIsDefaultAspect(int o)