Add video filtering support to VideoOutput

Add the QAbstractVideoFilter base class and integrate it with VideoOutput.

This can be used to perform arbitrary filtering or image processing
on the frames of a video stream of a VideoOutput element right before
the OpenGL texture is provided to the scenegraph by the video node.

This opens up the possibility to integrate computer vision
frameworks or accelerated image processing with Qt Quick applications
that display video streams using Qt Multimedia.

Conceptually it is somewhat similar to QVideoProbe, this
approach however allows modifying the frame, in real time
with tight integration to the scenegraph node, and targets
Qt Quick meaning setting up the filter and processing the results
of the computations happen completely in QML.

[ChangeLog] Added QAbstractVideoFilter that serves as a base class for QML
video filtering elements that integrate compute, vision, and image processing
frameworks with VideoOutput.

Change-Id: Ice1483f8c2daec5a43536978627a7bbb64549480
Reviewed-by: Yoann Lopes <yoann.lopes@theqtcompany.com>

9 files changed, 203 insertions, 11 deletions

diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
index 9a9256868..c36fca67d 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
@@ -262,6 +262,12 @@ bool QDeclarativeVideoOutput::createBackend(QMediaService *service)
         m_backend->updateGeometry();
     }
 
+    if (m_backend) {
+        m_backend->clearFilters();
+        for (int i = 0; i < m_filters.count(); ++i)
+            m_backend->appendFilter(m_filters[i]);
+    }
+
     return backendAvailable;
 }
 
@@ -795,6 +801,12 @@ void QDeclarativeVideoOutput::itemChange(QQuickItem::ItemChange change,
         m_backend->itemChange(change, changeData);
 }
 
+void QDeclarativeVideoOutput::releaseResources()
+{
+    if (m_backend)
+        m_backend->releaseResources();
+}
+
 void QDeclarativeVideoOutput::geometryChanged(const QRectF &newGeometry, const QRectF &oldGeometry)
 {
     Q_UNUSED(newGeometry);
@@ -809,4 +821,54 @@ void QDeclarativeVideoOutput::geometryChanged(const QRectF &newGeometry, const Q
     _q_updateGeometry();
 }
 
+/*!
+    \qmlproperty list<object> QtMultimedia::VideoOutput::filters
+
+    This property holds the list of video filters that are run on the video
+    frames. The order of the filters in the list matches the order in which
+    they will be invoked on the video frames. The objects in the list must be
+    instances of a subclass of QAbstractVideoFilter.
+
+    \sa QAbstractVideoFilter
+*/
+
+QQmlListProperty<QAbstractVideoFilter> QDeclarativeVideoOutput::filters()
+{
+    return QQmlListProperty<QAbstractVideoFilter>(this, 0, filter_append, filter_count, filter_at, filter_clear);
+}
+
+void QDeclarativeVideoOutput::filter_append(QQmlListProperty<QAbstractVideoFilter> *property, QAbstractVideoFilter *value)
+{
+    QDeclarativeVideoOutput *self = static_cast<QDeclarativeVideoOutput *>(property->object);
+    self->m_filters.append(value);
+    if (self->m_backend)
+        self->m_backend->appendFilter(value);
+}
+
+int QDeclarativeVideoOutput::filter_count(QQmlListProperty<QAbstractVideoFilter> *property)
+{
+    QDeclarativeVideoOutput *self = static_cast<QDeclarativeVideoOutput *>(property->object);
+    return self->m_filters.count();
+}
+
+QAbstractVideoFilter *QDeclarativeVideoOutput::filter_at(QQmlListProperty<QAbstractVideoFilter> *property, int index)
+{
+    QDeclarativeVideoOutput *self = static_cast<QDeclarativeVideoOutput *>(property->object);
+    return self->m_filters.at(index);
+}
+
+void QDeclarativeVideoOutput::filter_clear(QQmlListProperty<QAbstractVideoFilter> *property)
+{
+    QDeclarativeVideoOutput *self = static_cast<QDeclarativeVideoOutput *>(property->object);
+    self->m_filters.clear();
+    if (self->m_backend)
+        self->m_backend->clearFilters();
+}
+
+void QDeclarativeVideoOutput::_q_invalidateSceneGraph()
+{
+    if (m_backend)
+        m_backend->invalidateSceneGraph();
+}
+
 QT_END_NAMESPACE
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp
index 657ef1767..bde416f44 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp
@@ -34,6 +34,7 @@
 
 #include "qdeclarativevideooutput_render_p.h"
 #include "qdeclarativevideooutput_p.h"
+#include <QtMultimedia/qabstractvideofilter.h>
 #include <QtMultimedia/qvideorenderercontrol.h>
 #include <QtMultimedia/qmediaservice.h>
 #include <QtCore/qloggingcategory.h>
@@ -41,6 +42,8 @@
 #include <private/qsgvideonode_p.h>
 
 #include <QtGui/QOpenGLContext>
+#include <QtQuick/QQuickWindow>
+#include <QtCore/QRunnable>
 
 QT_BEGIN_NAMESPACE
 
@@ -103,11 +106,79 @@ bool QDeclarativeVideoRendererBackend::init(QMediaService *service)
     return false;
 }
 
+void QDeclarativeVideoRendererBackend::appendFilter(QAbstractVideoFilter *filter)
+{
+    QMutexLocker lock(&m_frameMutex);
+    m_filters.append(Filter(filter));
+}
+
+void QDeclarativeVideoRendererBackend::clearFilters()
+{
+    QMutexLocker lock(&m_frameMutex);
+    scheduleDeleteFilterResources();
+    m_filters.clear();
+}
+
+class FilterRunnableDeleter : public QRunnable
+{
+public:
+    FilterRunnableDeleter(const QList<QVideoFilterRunnable *> &runnables) : m_runnables(runnables) { }
+    void run() Q_DECL_OVERRIDE {
+        foreach (QVideoFilterRunnable *runnable, m_runnables)
+            delete runnable;
+    }
+private:
+    QList<QVideoFilterRunnable *> m_runnables;
+};
+
+void QDeclarativeVideoRendererBackend::scheduleDeleteFilterResources()
+{
+    if (!q->window())
+        return;
+
+    QList<QVideoFilterRunnable *> runnables;
+    for (int i = 0; i < m_filters.count(); ++i) {
+        if (m_filters[i].runnable) {
+            runnables.append(m_filters[i].runnable);
+            m_filters[i].runnable = 0;
+        }
+    }
+
+    if (!runnables.isEmpty()) {
+        // Request the scenegraph to run our cleanup job on the render thread.
+        // The execution of our QRunnable may happen after the QML tree including the QAbstractVideoFilter instance is
+        // destroyed on the main thread so no references to it must be used during cleanup.
+        q->window()->scheduleRenderJob(new FilterRunnableDeleter(runnables), QQuickWindow::BeforeSynchronizingStage);
+    }
+}
+
+void QDeclarativeVideoRendererBackend::releaseResources()
+{
+    // Called on the gui thread when the window is closed or changed.
+    QMutexLocker lock(&m_frameMutex);
+    scheduleDeleteFilterResources();
+}
+
+void QDeclarativeVideoRendererBackend::invalidateSceneGraph()
+{
+    // Called on the render thread, e.g. when the context is lost.
+    QMutexLocker lock(&m_frameMutex);
+    for (int i = 0; i < m_filters.count(); ++i) {
+        if (m_filters[i].runnable) {
+            delete m_filters[i].runnable;
+            m_filters[i].runnable = 0;
+        }
+    }
+}
+
 void QDeclarativeVideoRendererBackend::itemChange(QQuickItem::ItemChange change,
                                       const QQuickItem::ItemChangeData &changeData)
 {
-    Q_UNUSED(change);
-    Q_UNUSED(changeData);
+    if (change == QQuickItem::ItemSceneChange) {
+        if (changeData.window)
+            QObject::connect(changeData.window, SIGNAL(sceneGraphInvalidated()),
+                             q, SLOT(_q_invalidateSceneGraph()), Qt::DirectConnection);
+    }
 }
 
 void QDeclarativeVideoRendererBackend::releaseSource()
@@ -216,8 +287,36 @@ QSGNode *QDeclarativeVideoRendererBackend::updatePaintNode(QSGNode *oldNode,
     }
 #endif
 
+    bool isFrameModified = false;
     if (m_frameChanged) {
-        if (videoNode && videoNode->pixelFormat() != m_frame.pixelFormat()) {
+        // Run the VideoFilter if there is one. This must be done before potentially changing the videonode below.
+        if (m_frame.isValid() && !m_filters.isEmpty()) {
+            const QVideoSurfaceFormat surfaceFormat = videoSurface()->surfaceFormat();
+            for (int i = 0; i < m_filters.count(); ++i) {
+                QAbstractVideoFilter *filter = m_filters[i].filter;
+                QVideoFilterRunnable *&runnable = m_filters[i].runnable;
+                if (filter && filter->isActive()) {
+                    // Create the filter runnable if not yet done. Ownership is taken and is tied to this thread, on which rendering happens.
+                    if (!runnable)
+                        runnable = filter->createFilterRunnable();
+                    if (!runnable)
+                        continue;
+
+                    QVideoFilterRunnable::RunFlags flags = 0;
+                    if (i == m_filters.count() - 1)
+                        flags |= QVideoFilterRunnable::LastInChain;
+
+                    QVideoFrame newFrame = runnable->run(&m_frame, surfaceFormat, flags);
+
+                    if (newFrame.isValid() && newFrame != m_frame) {
+                        isFrameModified = true;
+                        m_frame = newFrame;
+                    }
+                }
+            }
+        }
+
+        if (videoNode && (videoNode->pixelFormat() != m_frame.pixelFormat() || videoNode->handleType() != m_frame.handleType())) {
             qCDebug(qLcVideo) << "updatePaintNode: deleting old video node because frame format changed";
             delete videoNode;
             videoNode = 0;
@@ -231,7 +330,9 @@ QSGNode *QDeclarativeVideoRendererBackend::updatePaintNode(QSGNode *oldNode,
 
         if (!videoNode) {
             foreach (QSGVideoNodeFactoryInterface* factory, m_videoNodeFactories) {
-                videoNode = factory->createNode(m_surface->surfaceFormat());
+                // Get a node that supports our frame. The surface is irrelevant, our
+                // QSGVideoItemSurface supports (logically) anything.
+                videoNode = factory->createNode(QVideoSurfaceFormat(m_frame.size(), m_frame.pixelFormat(), m_frame.handleType()));
                 if (videoNode) {
                     qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType()
                                      << " Supported formats for the handle by this node:"
@@ -252,7 +353,10 @@ QSGNode *QDeclarativeVideoRendererBackend::updatePaintNode(QSGNode *oldNode,
     videoNode->setTexturedRectGeometry(m_renderedRect, m_sourceTextureRect,
                                        qNormalizedOrientation(q->orientation()));
     if (m_frameChanged) {
-        videoNode->setCurrentFrame(m_frame);
+        QSGVideoNode::FrameFlags flags = 0;
+        if (isFrameModified)
+            flags |= QSGVideoNode::FrameFiltered;
+        videoNode->setCurrentFrame(m_frame, flags);
         //don't keep the frame for more than really necessary
         m_frameChanged = false;
         m_frame = QVideoFrame();
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h
index dd8449fa8..a0fed3d00 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_render_p.h
@@ -48,6 +48,8 @@ QT_BEGIN_NAMESPACE
 class QSGVideoItemSurface;
 class QVideoRendererControl;
 class QOpenGLContext;
+class QAbstractVideoFilter;
+class QVideoFilterRunnable;
 
 class QDeclarativeVideoRendererBackend : public QDeclarativeVideoBackend
 {
@@ -70,7 +72,14 @@ public:
     void present(const QVideoFrame &frame);
     void stop();
 
+    void appendFilter(QAbstractVideoFilter *filter) Q_DECL_OVERRIDE;
+    void clearFilters() Q_DECL_OVERRIDE;
+    void releaseResources() Q_DECL_OVERRIDE;
+    void invalidateSceneGraph() Q_DECL_OVERRIDE;
+
 private:
+    void scheduleDeleteFilterResources();
+
     QPointer<QVideoRendererControl> m_rendererControl;
     QList<QSGVideoNodeFactoryInterface*> m_videoNodeFactories;
     QSGVideoItemSurface *m_surface;
@@ -83,6 +92,14 @@ private:
     QMutex m_frameMutex;
     QRectF m_renderedRect;         // Destination pixel coordinates, clipped
     QRectF m_sourceTextureRect;    // Source texture coordinates
+
+    struct Filter {
+        Filter() : filter(0), runnable(0) { }
+        Filter(QAbstractVideoFilter *filter) : filter(filter), runnable(0) { }
+        QAbstractVideoFilter *filter;
+        QVideoFilterRunnable *runnable;
+    };
+    QList<Filter> m_filters;
 };
 
 class QSGVideoItemSurface : public QAbstractVideoSurface
diff --git a/src/qtmultimediaquicktools/qsgvideonode_i420.cpp b/src/qtmultimediaquicktools/qsgvideonode_i420.cpp
index 17b4924d0..3ff1f5bd9 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_i420.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_i420.cpp
@@ -311,7 +311,7 @@ QSGVideoNode_I420::~QSGVideoNode_I420()
 {
 }
 
-void QSGVideoNode_I420::setCurrentFrame(const QVideoFrame &frame)
+void QSGVideoNode_I420::setCurrentFrame(const QVideoFrame &frame, FrameFlags)
 {
     m_material->setCurrentFrame(frame);
     markDirty(DirtyMaterial);
diff --git a/src/qtmultimediaquicktools/qsgvideonode_i420.h b/src/qtmultimediaquicktools/qsgvideonode_i420.h
index 06d302d80..71dacf157 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_i420.h
+++ b/src/qtmultimediaquicktools/qsgvideonode_i420.h
@@ -49,7 +49,10 @@ public:
     virtual QVideoFrame::PixelFormat pixelFormat() const {
         return m_format.pixelFormat();
     }
-    void setCurrentFrame(const QVideoFrame &frame);
+    QAbstractVideoBuffer::HandleType handleType() const {
+        return QAbstractVideoBuffer::NoHandle;
+    }
+    void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags);
 
 private:
     void bindTexture(int id, int unit, int w, int h, const uchar *bits);
diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp b/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp
index 72cbc9614..486632914 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp
@@ -278,7 +278,7 @@ QSGVideoNode_RGB::~QSGVideoNode_RGB()
 {
 }
 
-void QSGVideoNode_RGB::setCurrentFrame(const QVideoFrame &frame)
+void QSGVideoNode_RGB::setCurrentFrame(const QVideoFrame &frame, FrameFlags)
 {
     m_material->setVideoFrame(frame);
     markDirty(DirtyMaterial);
diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb.h b/src/qtmultimediaquicktools/qsgvideonode_rgb.h
index e6800ce9d..6fcca9bf4 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_rgb.h
+++ b/src/qtmultimediaquicktools/qsgvideonode_rgb.h
@@ -50,7 +50,10 @@ public:
     virtual QVideoFrame::PixelFormat pixelFormat() const {
         return m_format.pixelFormat();
     }
-    void setCurrentFrame(const QVideoFrame &frame);
+    QAbstractVideoBuffer::HandleType handleType() const {
+        return QAbstractVideoBuffer::NoHandle;
+    }
+    void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags);
 
 private:
     QVideoSurfaceFormat m_format;
diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
index 3af14c0b8..b88327d0d 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
+++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp
@@ -235,7 +235,7 @@ QSGVideoNode_Texture::~QSGVideoNode_Texture()
 {
 }
 
-void QSGVideoNode_Texture::setCurrentFrame(const QVideoFrame &frame)
+void QSGVideoNode_Texture::setCurrentFrame(const QVideoFrame &frame, FrameFlags)
 {
     m_material->setVideoFrame(frame);
     markDirty(DirtyMaterial);
diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.h b/src/qtmultimediaquicktools/qsgvideonode_texture.h
index c5b84c53d..1389d4f3d 100644
--- a/src/qtmultimediaquicktools/qsgvideonode_texture.h
+++ b/src/qtmultimediaquicktools/qsgvideonode_texture.h
@@ -50,7 +50,10 @@ public:
     virtual QVideoFrame::PixelFormat pixelFormat() const {
         return m_format.pixelFormat();
     }
-    void setCurrentFrame(const QVideoFrame &frame);
+    QAbstractVideoBuffer::HandleType handleType() const {
+        return QAbstractVideoBuffer::GLTextureHandle;
+    }
+    void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags);
 
 private:
     QVideoSurfaceFormat m_format;