Add video filtering support to VideoOutput

Add the QAbstractVideoFilter base class and integrate it with VideoOutput.

This can be used to perform arbitrary filtering or image processing
on the frames of a video stream of a VideoOutput element right before
the OpenGL texture is provided to the scenegraph by the video node.

This opens up the possibility to integrate computer vision
frameworks or accelerated image processing with Qt Quick applications
that display video streams using Qt Multimedia.

Conceptually it is somewhat similar to QVideoProbe, this
approach however allows modifying the frame, in real time
with tight integration to the scenegraph node, and targets
Qt Quick meaning setting up the filter and processing the results
of the computations happen completely in QML.

[ChangeLog] Added QAbstractVideoFilter that serves as a base class for QML
video filtering elements that integrate compute, vision, and image processing
frameworks with VideoOutput.

Change-Id: Ice1483f8c2daec5a43536978627a7bbb64549480
Reviewed-by: Yoann Lopes <yoann.lopes@theqtcompany.com>

3 files changed, 29 insertions, 1 deletions

diff --git a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h
index 5996af773..1d267df82 100644
--- a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h
+++ b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_backend_p.h
@@ -46,6 +46,7 @@ QT_BEGIN_NAMESPACE
 class QAbstractVideoSurface;
 class QDeclarativeVideoOutput;
 class QMediaService;
+class QAbstractVideoFilter;
 
 class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoBackend
 {
@@ -70,6 +71,12 @@ public:
     // The viewport, adjusted for the pixel aspect ratio
     virtual QRectF adjustedViewport() const = 0;
 
+    virtual void appendFilter(QAbstractVideoFilter *filter) { Q_UNUSED(filter); }
+    virtual void clearFilters() { }
+
+    virtual void releaseResources() { }
+    virtual void invalidateSceneGraph() { }
+
 protected:
     QDeclarativeVideoOutput *q;
     QPointer<QMediaService> m_service;
diff --git a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h
index e1dbf5646..f022c0e3e 100644
--- a/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h
+++ b/src/multimedia/qtmultimediaquicktools_headers/qdeclarativevideooutput_p.h
@@ -40,6 +40,7 @@
 #include <QtQuick/qquickitem.h>
 #include <QtCore/qpointer.h>
 #include <QtMultimedia/qcamerainfo.h>
+#include <QtMultimedia/qabstractvideofilter.h>
 
 #include <private/qtmultimediaquickdefs_p.h>
 
@@ -60,6 +61,7 @@ class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoOutput : public QQuickItem
     Q_PROPERTY(bool autoOrientation READ autoOrientation WRITE setAutoOrientation NOTIFY autoOrientationChanged REVISION 2)
     Q_PROPERTY(QRectF sourceRect READ sourceRect NOTIFY sourceRectChanged)
     Q_PROPERTY(QRectF contentRect READ contentRect NOTIFY contentRectChanged)
+    Q_PROPERTY(QQmlListProperty<QAbstractVideoFilter> filters READ filters);
     Q_ENUMS(FillMode)
 
 public:
@@ -104,6 +106,8 @@ public:
     };
     SourceType sourceType() const;
 
+    QQmlListProperty<QAbstractVideoFilter> filters();
+
 Q_SIGNALS:
     void sourceChanged();
     void fillModeChanged(QDeclarativeVideoOutput::FillMode);
@@ -116,6 +120,7 @@ protected:
     QSGNode *updatePaintNode(QSGNode *, UpdatePaintNodeData *);
     void itemChange(ItemChange change, const ItemChangeData &changeData);
     void geometryChanged(const QRectF &newGeometry, const QRectF &oldGeometry);
+    void releaseResources();
 
 private Q_SLOTS:
     void _q_updateMediaObject();
@@ -123,10 +128,16 @@ private Q_SLOTS:
     void _q_updateNativeSize();
     void _q_updateGeometry();
     void _q_screenOrientationChanged(int);
+    void _q_invalidateSceneGraph();
 
 private:
     bool createBackend(QMediaService *service);
 
+    static void filter_append(QQmlListProperty<QAbstractVideoFilter> *property, QAbstractVideoFilter *value);
+    static int filter_count(QQmlListProperty<QAbstractVideoFilter> *property);
+    static QAbstractVideoFilter *filter_at(QQmlListProperty<QAbstractVideoFilter> *property, int index);
+    static void filter_clear(QQmlListProperty<QAbstractVideoFilter> *property);
+
     SourceType m_sourceType;
 
     QPointer<QObject> m_source;
@@ -145,6 +156,8 @@ private:
     QVideoOutputOrientationHandler *m_screenOrientationHandler;
 
     QScopedPointer<QDeclarativeVideoBackend> m_backend;
+
+    QList<QAbstractVideoFilter *> m_filters;
 };
 
 QT_END_NAMESPACE
diff --git a/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h b/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h
index 8be77ff07..9a15f4afa 100644
--- a/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h
+++ b/src/multimedia/qtmultimediaquicktools_headers/qsgvideonode_p.h
@@ -46,10 +46,16 @@ QT_BEGIN_NAMESPACE
 class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNode : public QSGGeometryNode
 {
 public:
+    enum FrameFlag {
+        FrameFiltered = 0x01
+    };
+    Q_DECLARE_FLAGS(FrameFlags, FrameFlag)
+
     QSGVideoNode();
 
-    virtual void setCurrentFrame(const QVideoFrame &frame) = 0;
+    virtual void setCurrentFrame(const QVideoFrame &frame, FrameFlags flags) = 0;
     virtual QVideoFrame::PixelFormat pixelFormat() const = 0;
+    virtual QAbstractVideoBuffer::HandleType handleType() const = 0;
 
     void setTexturedRectGeometry(const QRectF &boundingRect, const QRectF &textureRect, int orientation);
 
@@ -59,6 +65,8 @@ private:
     int m_orientation;
 };
 
+Q_DECLARE_OPERATORS_FOR_FLAGS(QSGVideoNode::FrameFlags)
+
 class Q_MULTIMEDIAQUICK_EXPORT QSGVideoNodeFactoryInterface
 {
 public: