Add subtitle support to VideoOutput

Implement generic subtitle support for the QML VideoOuput element
by rendering subtitles through a QQuickTextNode.

Change-Id: Ib6cd2763769891758fb1bb96964f30a38bdadefe
Reviewed-by: Samuel Mira <samuel.mira@qt.io>
Reviewed-by: Assam Boudjelthia <assam.boudjelthia@qt.io>
(cherry picked from commit d8ede46f1fcaf2322bb64daf48ad4565f7e0adfd)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

5 files changed, 74 insertions, 11 deletions

diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp
index 27129c770..0f46c4805 100644
--- a/src/multimedia/video/qvideotexturehelper.cpp
+++ b/src/multimedia/video/qvideotexturehelper.cpp
@@ -505,12 +505,12 @@ int updateRhiTextures(QVideoFrame frame, QRhi *rhi, QRhiResourceUpdateBatch *res
     return description->nplanes;
 }
 
-void SubtitleLayout::updateFromVideoFrame(const QVideoFrame &frame)
+bool SubtitleLayout::updateFromVideoFrame(const QVideoFrame &frame)
 {
     auto text = frame.subtitleText();
     text.replace(QLatin1Char('\n'), QChar::LineSeparator);
     if (layout.text() == text && videoSize == frame.size())
-        return;
+        return false;
 
     videoSize = frame.size();
     QFont font;
@@ -521,7 +521,7 @@ void SubtitleLayout::updateFromVideoFrame(const QVideoFrame &frame)
     layout.setText(text);
     if (text.isEmpty()) {
         bounds = {};
-        return;
+        return true;
     }
     layout.setFont(font);
     QTextOption option;
@@ -557,6 +557,7 @@ void SubtitleLayout::updateFromVideoFrame(const QVideoFrame &frame)
     textWidth += fontSize/4.;
 
     bounds = QRectF((videoSize.width() - textWidth)/2., y, textWidth, height);
+    return true;
 }
 
 void SubtitleLayout::draw(QPainter *painter, const QRectF &videoRect) const
diff --git a/src/multimedia/video/qvideotexturehelper_p.h b/src/multimedia/video/qvideotexturehelper_p.h
index 261d55cf0..168ce8a86 100644
--- a/src/multimedia/video/qvideotexturehelper_p.h
+++ b/src/multimedia/video/qvideotexturehelper_p.h
@@ -107,7 +107,7 @@ struct Q_MULTIMEDIA_EXPORT SubtitleLayout
     QRectF bounds;
     QTextLayout layout;
 
-    void updateFromVideoFrame(const QVideoFrame &frame);
+    bool updateFromVideoFrame(const QVideoFrame &frame);
     void draw(QPainter *painter, const QRectF &videoRect) const;
     QImage toImage() const;
 };
diff --git a/src/multimediaquick/qquickvideooutput.cpp b/src/multimediaquick/qquickvideooutput.cpp
index c625051f2..0c800f258 100644
--- a/src/multimediaquick/qquickvideooutput.cpp
+++ b/src/multimediaquick/qquickvideooutput.cpp
@@ -586,7 +586,7 @@ QSGNode *QQuickVideoOutput::updatePaintNode(QSGNode *oldNode,
             // Get a node that supports our frame. The surface is irrelevant, our
             // QSGVideoItemSurface supports (logically) anything.
             updateGeometry();
-            videoNode = new QSGVideoNode(m_surfaceFormat);
+            videoNode = new QSGVideoNode(this, m_surfaceFormat);
             qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType();
         }
     }
@@ -597,9 +597,6 @@ QSGNode *QQuickVideoOutput::updatePaintNode(QSGNode *oldNode,
         return nullptr;
     }
 
-    // Negative rotations need lots of %360
-    videoNode->setTexturedRectGeometry(m_renderedRect, m_sourceTextureRect,
-                                       qNormalizedOrientation(orientation()));
     if (m_frameChanged) {
         videoNode->setCurrentFrame(m_frame);
 
@@ -607,6 +604,11 @@ QSGNode *QQuickVideoOutput::updatePaintNode(QSGNode *oldNode,
         m_frameChanged = false;
         m_frame = QVideoFrame();
     }
+
+    // Negative rotations need lots of %360
+    videoNode->setTexturedRectGeometry(m_renderedRect, m_sourceTextureRect,
+                                       qNormalizedOrientation(orientation()));
+
     return videoNode;
 }
 
diff --git a/src/multimediaquick/qsgvideonode_p.cpp b/src/multimediaquick/qsgvideonode_p.cpp
index 968fe374d..d3f40b54b 100644
--- a/src/multimediaquick/qsgvideonode_p.cpp
+++ b/src/multimediaquick/qsgvideonode_p.cpp
@@ -41,6 +41,8 @@
 #include <QtQuick/qsgmaterial.h>
 #include "qsgvideotexture_p.h"
 #include <QtMultimedia/private/qvideotexturehelper_p.h>
+#include <private/qquicktextnode_p.h>
+#include <private/qquickvideooutput_p.h>
 #include <qmutex.h>
 
 QT_BEGIN_NAMESPACE
@@ -120,6 +122,8 @@ void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &tex
         setGeometry(g);
 
     markDirty(DirtyGeometry);
+
+    setSubtitleGeometry();
 }
 
 class QSGVideoMaterial;
@@ -271,8 +275,9 @@ QSGVideoMaterial::QSGVideoMaterial(const QVideoFrameFormat &format) :
     setFlag(Blending, false);
 }
 
-QSGVideoNode::QSGVideoNode(const QVideoFrameFormat &format)
-    : m_orientation(-1),
+QSGVideoNode::QSGVideoNode(QQuickVideoOutput *parent, const QVideoFrameFormat &format)
+    : m_parent(parent),
+    m_orientation(-1),
     m_format(format)
 {
     setFlag(QSGNode::OwnsMaterial);
@@ -280,10 +285,54 @@ QSGVideoNode::QSGVideoNode(const QVideoFrameFormat &format)
     setMaterial(m_material);
 }
 
+QSGVideoNode::~QSGVideoNode()
+{
+    delete m_subtitleTextNode;
+}
+
 void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame)
 {
     m_material->setCurrentFrame(frame);
     markDirty(DirtyMaterial);
+    updateSubtitle(frame);
+}
+
+void QSGVideoNode::updateSubtitle(const QVideoFrame &frame)
+{
+    if (!m_subtitleLayout.updateFromVideoFrame(frame))
+        return;
+
+    delete m_subtitleTextNode;
+    m_subtitleTextNode = nullptr;
+    if (frame.subtitleText().isEmpty())
+        return;
+
+    m_subtitleTextNode = new QQuickTextNode(m_parent);
+    QColor bgColor = Qt::black;
+    bgColor.setAlpha(128);
+    m_subtitleTextNode->addRectangleNode(m_subtitleLayout.bounds, bgColor);
+    m_subtitleTextNode->addTextLayout(m_subtitleLayout.layout.position(), &m_subtitleLayout.layout, Qt::white);
+    appendChildNode(m_subtitleTextNode);
+    setSubtitleGeometry();
 }
 
+void QSGVideoNode::setSubtitleGeometry()
+{
+    if (!m_subtitleTextNode)
+        return;
+
+    QSizeF s = m_subtitleLayout.videoSize;
+    QMatrix4x4 transform = {
+        float(m_rect.width()/s.width()), 0, 0, float(m_rect.x()),
+        0, float(m_rect.height()/s.height()), 0, float(m_rect.y()),
+        0, 0, 1, 0,
+        0, 0, 0, 1
+    };
+    transform.rotate(m_orientation/180*M_PI, 0, 0, 1);
+    m_subtitleTextNode->setMatrix(transform);
+    m_subtitleTextNode->markDirty(DirtyGeometry);
+}
+
+
+
 QT_END_NAMESPACE
diff --git a/src/multimediaquick/qsgvideonode_p.h b/src/multimediaquick/qsgvideonode_p.h
index 4b26be28a..d62248b7c 100644
--- a/src/multimediaquick/qsgvideonode_p.h
+++ b/src/multimediaquick/qsgvideonode_p.h
@@ -53,6 +53,7 @@
 
 #include <QtQuick/qsgnode.h>
 #include <private/qtmultimediaquickglobal_p.h>
+#include "private/qvideotexturehelper_p.h"
 
 #include <QtMultimedia/qvideoframe.h>
 #include <QtMultimedia/qvideoframeformat.h>
@@ -61,11 +62,14 @@
 QT_BEGIN_NAMESPACE
 
 class QSGVideoMaterial;
+class QQuickVideoOutput;
+class QQuickTextNode;
 
 class QSGVideoNode : public QSGGeometryNode
 {
 public:
-    QSGVideoNode(const QVideoFrameFormat &format);
+    QSGVideoNode(QQuickVideoOutput *parent, const QVideoFrameFormat &format);
+    ~QSGVideoNode();
 
     QVideoFrameFormat::PixelFormat pixelFormat() const {
         return m_format.pixelFormat();
@@ -75,12 +79,19 @@ public:
     void setTexturedRectGeometry(const QRectF &boundingRect, const QRectF &textureRect, int orientation);
 
 private:
+    void updateSubtitle(const QVideoFrame &frame);
+    void setSubtitleGeometry();
+
+    QQuickVideoOutput *m_parent = nullptr;
     QRectF m_rect;
     QRectF m_textureRect;
     int m_orientation;
 
     QVideoFrameFormat m_format;
     QSGVideoMaterial *m_material;
+
+    QVideoTextureHelper::SubtitleLayout m_subtitleLayout;
+    QQuickTextNode *m_subtitleTextNode = nullptr;
 };
 
 QT_END_NAMESPACE