Expose video surfaces in rendering components

* Currently QVideoWidget, QGraphicsVideoItem and QML Video Output
tightly depend on QMediaPlayer, QMediaService etc.

Since QMediaService is deprecated and will be removed from Qt6,
its usage must be removed from rendering components.

* Also it is a part of a task to make all components independent,
e.g. without dependencies to another components, like the media player,
or video producer.

These video consumer components should be reusable without QMediaPlayer too,
which today is not possible,
and should have responsibility for a single part of the functionality.

* It is a part of a task to allow users to decide how to render video,
using either video surface or native windows.

Introducing
 QVideoWidget::videoSurface()
 QGraphicsVideoItem::videoSurface()
 QDeclarativeVideoOutput::videoSurface()

that create a surface, if necessary, and returns the underlying video surface
which provides a possibility to render video frames without knowledge of any services and controls.

Can be used like:
  QMediaPlayer->setVideoOutput(QVideoWidget->videoSurface());
or
  QMediaPlayer->setVideoOutput(QGraphicsVideoItem->videoSurface());

This allows QMediaPlayer to get video frames from a backend,
without using QVideoWidget/QGraphicsVideoItem, and to present these frames to provided surface.

Moreover, now it is possible to render video frames without QMediaPlayer:
  QVideoSurfaceFormat format(img.size(), QVideoFrame::Format_ARGB32);
  videoWidget->videoSurface()->start(format);
  videoWidget->videoSurface()->present(img);

[ChangeLog] Introduced videoSurface property to QVideoWidget, QGraphicsVideoItem and
QDeclarativeVideoOutput.

Task-number: QTBUG-80431
Change-Id: I49641750537160832e29c297279e72b8288e974c
Reviewed-by: Paul Wicking <paul.wicking@qt.io>

10 files changed, 126 insertions, 29 deletions

diff --git a/src/imports/multimedia/multimedia.cpp b/src/imports/multimedia/multimedia.cpp
index 8aed83d33..6106bb45d 100644
--- a/src/imports/multimedia/multimedia.cpp
+++ b/src/imports/multimedia/multimedia.cpp
@@ -60,6 +60,7 @@
 #include "qdeclarativecameraimageprocessing_p.h"
 #include "qdeclarativecameraviewfinder_p.h"
 #include "qdeclarativetorch_p.h"
+#include <QAbstractVideoSurface>
 
 QML_DECLARE_TYPE(QSoundEffect)
 
@@ -160,6 +161,10 @@ public:
         // 5.13 types
         qmlRegisterType<QDeclarativeVideoOutput, 13>(uri, 5, 13, "VideoOutput");
 
+        // 5.15 types
+        qmlRegisterType<QDeclarativeVideoOutput, 15>(uri, 5, 15, "VideoOutput");
+        qmlRegisterType<QAbstractVideoSurface>();
+
         // Auto-increment the import to stay in sync with ALL future QtQuick minor versions from 5.11 onward
         qmlRegisterModule(uri, 5, QT_VERSION_MINOR);
     }
diff --git a/src/multimedia/doc/snippets/multimedia-snippets/video.cpp b/src/multimedia/doc/snippets/multimedia-snippets/video.cpp
index 3c14f7009..46327e3d6 100644
--- a/src/multimedia/doc/snippets/multimedia-snippets/video.cpp
+++ b/src/multimedia/doc/snippets/multimedia-snippets/video.cpp
@@ -117,6 +117,7 @@ public:
     void VideoWidget();
     void VideoWindowControl();
     void VideoWidgetControl();
+    void VideoSurface();
 
 private:
     // Common naming
@@ -163,6 +164,28 @@ void VideoExample::VideoWidget()
     //! [Setting surface in player]
 }
 
+void VideoExample::VideoSurface()
+{
+    //! [Widget Surface]
+    QImage img = QImage("images/qt-logo.png").convertToFormat(QImage::Format_ARGB32);
+    QVideoSurfaceFormat format(img.size(), QVideoFrame::Format_ARGB32);
+    videoWidget = new QVideoWidget;
+    videoWidget->videoSurface()->start(format);
+    videoWidget->videoSurface()->present(img);
+    videoWidget->show();
+    //! [Widget Surface]
+
+    //! [GraphicsVideoItem Surface]
+    QGraphicsVideoItem *item = new QGraphicsVideoItem;
+    graphicsView->scene()->addItem(item);
+    graphicsView->show();
+    QImage img = QImage("images/qt-logo.png").convertToFormat(QImage::Format_ARGB32);
+    QVideoSurfaceFormat format(img.size(), QVideoFrame::Format_ARGB32);
+    item->videoSurface()->start(format);
+    item->videoSurface()->present(img);
+    //! [GraphicsVideoItem Surface]
+}
+
 void VideoExample::VideoWidgetControl()
 {
     //! [Video widget control]
diff --git a/src/multimediawidgets/qgraphicsvideoitem.cpp b/src/multimediawidgets/qgraphicsvideoitem.cpp
index 2db8987fb..fae1f9048 100644
--- a/src/multimediawidgets/qgraphicsvideoitem.cpp
+++ b/src/multimediawidgets/qgraphicsvideoitem.cpp
@@ -232,6 +232,22 @@ QMediaObject *QGraphicsVideoItem::mediaObject() const
 }
 
 /*!
+    \since 5.15
+    \property QGraphicsVideoItem::videoSurface
+    \brief Returns the underlying video surface that can render video frames
+    to the current item.
+    This property is never \c nullptr.
+    Example of how to render video frames to QGraphicsVideoItem:
+    \snippet multimedia-snippets/video.cpp GraphicsVideoItem Surface
+    \sa QMediaPlayer::setVideoOutput
+*/
+
+QAbstractVideoSurface *QGraphicsVideoItem::videoSurface() const
+{
+    return d_func()->surface;
+}
+
+/*!
   \internal
 */
 bool QGraphicsVideoItem::setMediaObject(QMediaObject *object)
diff --git a/src/multimediawidgets/qgraphicsvideoitem.h b/src/multimediawidgets/qgraphicsvideoitem.h
index 5aa3bd75c..de73458fd 100644
--- a/src/multimediawidgets/qgraphicsvideoitem.h
+++ b/src/multimediawidgets/qgraphicsvideoitem.h
@@ -60,11 +60,13 @@ class Q_MULTIMEDIAWIDGETS_EXPORT QGraphicsVideoItem : public QGraphicsObject, pu
     Q_PROPERTY(QPointF offset READ offset WRITE setOffset)
     Q_PROPERTY(QSizeF size READ size WRITE setSize)
     Q_PROPERTY(QSizeF nativeSize READ nativeSize NOTIFY nativeSizeChanged)
+    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface)
 public:
     explicit QGraphicsVideoItem(QGraphicsItem *parent = nullptr);
     ~QGraphicsVideoItem();
 
     QMediaObject *mediaObject() const override;
+    QAbstractVideoSurface *videoSurface() const;
 
     Qt::AspectRatioMode aspectRatioMode() const;
     void setAspectRatioMode(Qt::AspectRatioMode mode);
diff --git a/src/multimediawidgets/qpaintervideosurface.cpp b/src/multimediawidgets/qpaintervideosurface.cpp
index 944ea23a7..1c8e2dd63 100644
--- a/src/multimediawidgets/qpaintervideosurface.cpp
+++ b/src/multimediawidgets/qpaintervideosurface.cpp
@@ -1473,29 +1473,28 @@ void QPainterVideoSurface::stop()
 bool QPainterVideoSurface::present(const QVideoFrame &frame)
 {
     if (!m_ready) {
-        if (!isActive())
+        if (!isActive()) {
             setError(StoppedError);
+            return false;
+        }
     } else if (frame.isValid()
             && (frame.pixelFormat() != m_pixelFormat || frame.size() != m_frameSize)) {
         setError(IncorrectFormatError);
 
         stop();
+        return false;
     } else {
         QAbstractVideoSurface::Error error = m_painter->setCurrentFrame(frame);
-
         if (error != QAbstractVideoSurface::NoError) {
             setError(error);
-
             stop();
-        } else {
-            m_ready = false;
-
-            emit frameChanged();
-
-            return true;
+            return false;
         }
+
+        m_ready = false;
+        emit frameChanged();
     }
-    return false;
+    return true;
 }
 
 /*!
diff --git a/src/multimediawidgets/qvideowidget.cpp b/src/multimediawidgets/qvideowidget.cpp
index 5adc6c359..723a9bbb3 100644
--- a/src/multimediawidgets/qvideowidget.cpp
+++ b/src/multimediawidgets/qvideowidget.cpp
@@ -145,7 +145,8 @@ QRendererVideoWidgetBackend::QRendererVideoWidgetBackend(
     connect(m_surface, SIGNAL(surfaceFormatChanged(QVideoSurfaceFormat)),
             this, SLOT(formatChanged(QVideoSurfaceFormat)));
 
-    m_rendererControl->setSurface(m_surface);
+    if (m_rendererControl)
+        m_rendererControl->setSurface(m_surface);
 }
 
 QRendererVideoWidgetBackend::~QRendererVideoWidgetBackend()
@@ -153,14 +154,21 @@ QRendererVideoWidgetBackend::~QRendererVideoWidgetBackend()
     delete m_surface;
 }
 
+QAbstractVideoSurface *QRendererVideoWidgetBackend::videoSurface() const
+{
+    return m_surface;
+}
+
 void QRendererVideoWidgetBackend::releaseControl()
 {
-    m_service->releaseControl(m_rendererControl);
+    if (m_service && m_rendererControl)
+        m_service->releaseControl(m_rendererControl);
 }
 
 void QRendererVideoWidgetBackend::clearSurface()
 {
-    m_rendererControl->setSurface(0);
+    if (m_rendererControl)
+        m_rendererControl->setSurface(0);
 }
 
 void QRendererVideoWidgetBackend::setBrightness(int brightness)
@@ -469,7 +477,7 @@ void QVideoWidgetPrivate::clearService()
 
             delete rendererBackend;
             rendererBackend = 0;
-        } else {
+        } else if (windowBackend) {
             windowBackend->releaseControl();
 
             delete windowBackend;
@@ -515,18 +523,15 @@ bool QVideoWidgetPrivate::createWindowBackend()
 
 bool QVideoWidgetPrivate::createRendererBackend()
 {
-    if (QMediaControl *control = service->requestControl(QVideoRendererControl_iid)) {
-        if (QVideoRendererControl *rendererControl = qobject_cast<QVideoRendererControl *>(control)) {
-            rendererBackend = new QRendererVideoWidgetBackend(service, rendererControl, q_func());
-            currentBackend = rendererBackend;
-
-            setCurrentControl(rendererBackend);
+    QMediaControl *control = service
+                           ? service->requestControl(QVideoRendererControl_iid)
+                           : nullptr;
+    rendererBackend = new QRendererVideoWidgetBackend(service,
+        qobject_cast<QVideoRendererControl *>(control), q_func());
+    currentBackend = rendererBackend;
+    setCurrentControl(rendererBackend);
 
-            return true;
-        }
-        service->releaseControl(control);
-    }
-    return false;
+    return !service || (service && control);
 }
 
 void QVideoWidgetPrivate::_q_serviceDestroyed()
@@ -696,6 +701,29 @@ bool QVideoWidget::setMediaObject(QMediaObject *object)
 }
 
 /*!
+    \since 5.15
+    \property QVideoWidget::videoSurface
+    \brief Returns the underlaying video surface that can render video frames
+    to the current widget.
+    This property is never \c nullptr.
+    Example of how to render video frames to QVideoWidget:
+    \snippet multimedia-snippets/video.cpp Widget Surface
+    \sa QMediaPlayer::setVideoOutput
+*/
+
+QAbstractVideoSurface *QVideoWidget::videoSurface() const
+{
+    auto d = const_cast<QVideoWidgetPrivate *>(d_func());
+
+    if (!d->rendererBackend) {
+        d->clearService();
+        d->createRendererBackend();
+    }
+
+    return d->rendererBackend->videoSurface();
+}
+
+/*!
     \property QVideoWidget::aspectRatioMode
     \brief how video is scaled with respect to its aspect ratio.
 */
diff --git a/src/multimediawidgets/qvideowidget.h b/src/multimediawidgets/qvideowidget.h
index fff1153ca..c5abe2ce3 100644
--- a/src/multimediawidgets/qvideowidget.h
+++ b/src/multimediawidgets/qvideowidget.h
@@ -51,6 +51,7 @@ QT_BEGIN_NAMESPACE
 class QMediaObject;
 
 class QVideoWidgetPrivate;
+class QAbstractVideoSurface;
 class Q_MULTIMEDIAWIDGETS_EXPORT QVideoWidget : public QWidget, public QMediaBindableInterface
 {
     Q_OBJECT
@@ -62,12 +63,14 @@ class Q_MULTIMEDIAWIDGETS_EXPORT QVideoWidget : public QWidget, public QMediaBin
     Q_PROPERTY(int contrast READ contrast WRITE setContrast NOTIFY contrastChanged)
     Q_PROPERTY(int hue READ hue WRITE setHue NOTIFY hueChanged)
     Q_PROPERTY(int saturation READ saturation WRITE setSaturation NOTIFY saturationChanged)
+    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface)
 
 public:
     explicit QVideoWidget(QWidget *parent = nullptr);
     ~QVideoWidget();
 
     QMediaObject *mediaObject() const override;
+    QAbstractVideoSurface *videoSurface() const;
 
 #ifdef Q_QDOC
     bool isFullScreen() const;
diff --git a/src/multimediawidgets/qvideowidget_p.h b/src/multimediawidgets/qvideowidget_p.h
index 07912ee90..1db91f3f9 100644
--- a/src/multimediawidgets/qvideowidget_p.h
+++ b/src/multimediawidgets/qvideowidget_p.h
@@ -131,6 +131,8 @@ public:
     QRendererVideoWidgetBackend(QMediaService *service, QVideoRendererControl *control, QWidget *widget);
     ~QRendererVideoWidgetBackend();
 
+    QAbstractVideoSurface *videoSurface() const;
+
     void releaseControl();
     void clearSurface();
 
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
index 32166502d..128d0a616 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput.cpp
@@ -138,6 +138,7 @@ QDeclarativeVideoOutput::QDeclarativeVideoOutput(QQuickItem *parent) :
 {
     initResource();
     setFlag(ItemHasContents, true);
+    m_backend.reset(new QDeclarativeVideoRendererBackend(this));
 }
 
 QDeclarativeVideoOutput::~QDeclarativeVideoOutput()
@@ -148,6 +149,22 @@ QDeclarativeVideoOutput::~QDeclarativeVideoOutput()
 }
 
 /*!
+    \qmlproperty object QtMultimedia::VideoOutput::videoSurface
+    \since 5.15
+
+    This property holds the underlaying video surface that can be used
+    to render the video frames to this VideoOutput element.
+    It is similar to setting a QObject with \c videoSurface property as a source,
+    where this video surface will be set.
+    \sa setSource
+*/
+
+QAbstractVideoSurface *QDeclarativeVideoOutput::videoSurface() const
+{
+    return m_backend->videoSurface();
+}
+
+/*!
     \qmlproperty variant QtMultimedia::VideoOutput::source
 
     This property holds the source item providing the video frames like MediaPlayer or Camera.
@@ -251,7 +268,8 @@ bool QDeclarativeVideoOutput::createBackend(QMediaService *service)
     }
 #if QT_CONFIG(opengl)
     if (!backendAvailable) {
-        m_backend.reset(new QDeclarativeVideoRendererBackend(this));
+        if (!m_backend)
+            m_backend.reset(new QDeclarativeVideoRendererBackend(this));
         if (m_backend->init(service))
             backendAvailable = true;
     }
@@ -293,9 +311,6 @@ void QDeclarativeVideoOutput::_q_updateMediaObject()
     if (m_mediaObject.data() == mediaObject)
         return;
 
-    if (m_sourceType != VideoSurfaceSource)
-        m_backend.reset();
-
     m_mediaObject.clear();
     m_service.clear();
 
diff --git a/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h b/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h
index 8ea0dc338..9e826d0ff 100644
--- a/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h
+++ b/src/qtmultimediaquicktools/qdeclarativevideooutput_p.h
@@ -67,6 +67,7 @@ class QMediaObject;
 class QMediaService;
 class QDeclarativeVideoBackend;
 class QVideoOutputOrientationHandler;
+class QAbstractVideoSurface;
 
 class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoOutput : public QQuickItem
 {
@@ -80,6 +81,7 @@ class Q_MULTIMEDIAQUICK_EXPORT QDeclarativeVideoOutput : public QQuickItem
     Q_PROPERTY(QRectF contentRect READ contentRect NOTIFY contentRectChanged)
     Q_PROPERTY(QQmlListProperty<QAbstractVideoFilter> filters READ filters);
     Q_PROPERTY(FlushMode flushMode READ flushMode WRITE setFlushMode NOTIFY flushModeChanged REVISION 13)
+    Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface REVISION 15)
     Q_ENUMS(FlushMode)
     Q_ENUMS(FillMode)
 
@@ -102,6 +104,8 @@ public:
     QDeclarativeVideoOutput(QQuickItem *parent = 0);
     ~QDeclarativeVideoOutput();
 
+    QAbstractVideoSurface *videoSurface() const;
+
     QObject *source() const { return m_source.data(); }
     void setSource(QObject *source);