Implement platform independent subtitle rendering

Add support for rendering subtitles in a platform independent way
using our own text rendering infrastructure.

Implement support for subtitle rendering in QVideoFrame::paint().
Add a flag to disable subtitle the rendering if not desired.

Support for Qt Quick VideoOuput is still missing, and will come in
a follow-up change.

Implement setting the subtitle text correctly on macOS/iOS. Other
platforms will be done in follow-up changes.

Pick-to: 6.2
Change-Id: If5c689d4919d7a8df23399184f6e724028b0e980
Reviewed-by: Samuel Mira <samuel.mira@qt.io>
Reviewed-by: Assam Boudjelthia <assam.boudjelthia@qt.io>

8 files changed, 257 insertions, 15 deletions

diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 400e122ab..71e16162c 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -44,6 +44,7 @@
 #include "qvideoframeconversionhelper_p.h"
 #include "qvideoframeformat.h"
 #include "qpainter.h"
+#include <qtextlayout.h>
 
 #include <qimage.h>
 #include <qmutex.h>
@@ -114,7 +115,7 @@ public:
     QAbstractVideoBuffer *buffer = nullptr;
     int mappedCount = 0;
     QMutex mapMutex;
-    QVariantMap metadata;
+    QString subtitleText;
 
 private:
     Q_DISABLE_COPY(QVideoFramePrivate)
@@ -715,6 +716,22 @@ QImage QVideoFrame::toImage() const
 }
 
 /*!
+    Returns the subtitle text that should be rendered together with this video frame.
+*/
+QString QVideoFrame::subtitleText() const
+{
+    return d->subtitleText;
+}
+
+/*!
+    Sets the subtitle text that should be rendered together with this video frame to \a text.
+*/
+void QVideoFrame::setSubtitleText(const QString &text)
+{
+    d->subtitleText = text;
+}
+
+/*!
     Uses a QPainter, \a{painter}, to render this QVideoFrame to \a rect.
     The PaintOptions \a options can be used to specify a background color and
     how \a rect should be filled with the video.
@@ -725,7 +742,7 @@ QImage QVideoFrame::toImage() const
 void QVideoFrame::paint(QPainter *painter, const QRectF &rect, const PaintOptions &options)
 {
     if (!isValid()) {
-        painter->fillRect(rect, painter->background());
+        painter->fillRect(rect, options.backgroundColor);
         return;
     }
 
@@ -793,6 +810,17 @@ void QVideoFrame::paint(QPainter *painter, const QRectF &rect, const PaintOption
     } else {
         painter->fillRect(rect, Qt::black);
     }
+
+    if ((options.paintFlags & PaintOptions::DontDrawSubtitles) || d->subtitleText.isEmpty())
+        return;
+
+    // draw subtitles
+    auto text = d->subtitleText;
+    text.replace(QLatin1Char('\n'), QChar::LineSeparator);
+
+    QVideoTextureHelper::SubtitleLayout layout;
+    layout.updateFromVideoFrame(*this);
+    layout.draw(painter, targetRect);
 }
 
 #ifndef QT_NO_DEBUG_STREAM
diff --git a/src/multimedia/video/qvideoframe.h b/src/multimedia/video/qvideoframe.h
index 6a622070c..2bf2bd5ef 100644
--- a/src/multimedia/video/qvideoframe.h
+++ b/src/multimedia/video/qvideoframe.h
@@ -131,8 +131,16 @@ public:
     struct PaintOptions {
         QColor backgroundColor = Qt::transparent;
         Qt::AspectRatioMode aspectRatioMode = Qt::KeepAspectRatio;
+        enum PaintFlag {
+            DontDrawSubtitles = 0x1
+        };
+        Q_DECLARE_FLAGS(PaintFlags, PaintFlag)
+        PaintFlags paintFlags = {};
     };
 
+    QString subtitleText() const;
+    void setSubtitleText(const QString &text);
+
     void paint(QPainter *painter, const QRectF &rect, const PaintOptions &options);
 
     QVideoFrame(QAbstractVideoBuffer *buffer, const QVideoFrameFormat &format);
diff --git a/src/multimedia/video/qvideosink.cpp b/src/multimedia/video/qvideosink.cpp
index 78943c6e6..94a153564 100644
--- a/src/multimedia/video/qvideosink.cpp
+++ b/src/multimedia/video/qvideosink.cpp
@@ -158,6 +158,14 @@ QPlatformVideoSink *QVideoSink::platformVideoSink() const
 }
 
 /*!
+    Returns the current subtitle text.
+ */
+QString QVideoSink::subtitleText() const
+{
+    return d->videoSink->subtitleText();
+}
+
+/*!
     Returns the size of the video currently being played back. If no video is
     being played, this method returns an invalid size.
  */
diff --git a/src/multimedia/video/qvideosink.h b/src/multimedia/video/qvideosink.h
index 6d4cf34ea..d836abb4e 100644
--- a/src/multimedia/video/qvideosink.h
+++ b/src/multimedia/video/qvideosink.h
@@ -68,8 +68,11 @@ public:
 
     QPlatformVideoSink *platformVideoSink() const;
 
+    QString subtitleText() const;
+
 Q_SIGNALS:
     void newVideoFrame(const QVideoFrame &frame) const;
+    void subtitleTextChanged(const QString &subtitleText) const;
 
     void videoSizeChanged();
 
diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp
index 3b731182c..27129c770 100644
--- a/src/multimedia/video/qvideotexturehelper.cpp
+++ b/src/multimedia/video/qvideotexturehelper.cpp
@@ -42,6 +42,9 @@
 #ifdef Q_OS_ANDROID
 #include <private/qandroidvideooutput_p.h>
 #endif
+
+#include <qpainter.h>
+
 QT_BEGIN_NAMESPACE
 
 namespace QVideoTextureHelper
@@ -502,6 +505,101 @@ int updateRhiTextures(QVideoFrame frame, QRhi *rhi, QRhiResourceUpdateBatch *res
     return description->nplanes;
 }
 
+void SubtitleLayout::updateFromVideoFrame(const QVideoFrame &frame)
+{
+    auto text = frame.subtitleText();
+    text.replace(QLatin1Char('\n'), QChar::LineSeparator);
+    if (layout.text() == text && videoSize == frame.size())
+        return;
+
+    videoSize = frame.size();
+    QFont font;
+    // 0.045 - based on this https://www.md-subs.com/saa-subtitle-font-size
+    qreal fontSize = videoSize.height() * 0.045;
+    font.setPointSize(fontSize);
+
+    layout.setText(text);
+    if (text.isEmpty()) {
+        bounds = {};
+        return;
+    }
+    layout.setFont(font);
+    QTextOption option;
+    option.setUseDesignMetrics(true);
+    option.setAlignment(Qt::AlignCenter);
+    layout.setTextOption(option);
+
+    QFontMetrics metrics(font);
+    int leading = metrics.leading();
+
+    qreal lineWidth = videoSize.width()*.9;
+    qreal margin = videoSize.width()*.05;
+    qreal height = 0;
+    qreal textWidth = 0;
+    layout.beginLayout();
+    while (1) {
+        QTextLine line = layout.createLine();
+        if (!line.isValid())
+            break;
+
+        line.setLineWidth(lineWidth);
+        height += leading;
+        line.setPosition(QPointF(margin, height));
+        height += line.height();
+        textWidth = qMax(textWidth, line.naturalTextWidth());
+    }
+    layout.endLayout();
+
+    // put subtitles vertically in lower part of the video but not stuck to the bottom
+    int bottomMargin = videoSize.height() / 20;
+    qreal y = videoSize.height() - bottomMargin - height;
+    layout.setPosition(QPointF(0, y));
+    textWidth += fontSize/4.;
+
+    bounds = QRectF((videoSize.width() - textWidth)/2., y, textWidth, height);
+}
+
+void SubtitleLayout::draw(QPainter *painter, const QRectF &videoRect) const
+{
+    painter->save();
+    painter->translate(videoRect.topLeft());
+    painter->scale(videoRect.width()/videoSize.width(), videoRect.height()/videoSize.height());
+    painter->setCompositionMode(QPainter::CompositionMode_SourceOver);
+
+    QColor bgColor = Qt::black;
+    bgColor.setAlpha(128);
+    painter->setBrush(bgColor);
+    painter->setPen(Qt::NoPen);
+    painter->drawRect(bounds);
+
+    QTextLayout::FormatRange range;
+    range.start = 0;
+    range.length = layout.text().size();
+    range.format.setForeground(Qt::white);
+    layout.draw(painter, {}, { range });
+    painter->restore();
+}
+
+QImage SubtitleLayout::toImage() const
+{
+    auto size = bounds.size().toSize();
+    if (size.isEmpty())
+        return QImage();
+    QImage img(size, QImage::Format_RGBA8888_Premultiplied);
+    QColor bgColor = Qt::black;
+    bgColor.setAlpha(128);
+    img.fill(bgColor);
+
+    QPainter painter(&img);
+    painter.translate(-bounds.topLeft());
+    QTextLayout::FormatRange range;
+    range.start = 0;
+    range.length = layout.text().size();
+    range.format.setForeground(Qt::white);
+    layout.draw(&painter, {}, { range });
+    return img;
+}
+
 }
 
 QT_END_NAMESPACE
diff --git a/src/multimedia/video/qvideotexturehelper_p.h b/src/multimedia/video/qvideotexturehelper_p.h
index c6256efeb..261d55cf0 100644
--- a/src/multimedia/video/qvideotexturehelper_p.h
+++ b/src/multimedia/video/qvideotexturehelper_p.h
@@ -54,9 +54,12 @@
 #include <qvideoframeformat.h>
 #include <private/qrhi_p.h>
 
+#include <QtGui/qtextlayout.h>
+
 QT_BEGIN_NAMESPACE
 
 class QVideoFrame;
+class QTextLayout;
 
 namespace QVideoTextureHelper
 {
@@ -98,6 +101,17 @@ Q_MULTIMEDIA_EXPORT void updateUniformData(QByteArray *dst, const QVideoFrameFor
 Q_MULTIMEDIA_EXPORT int updateRhiTextures(QVideoFrame frame, QRhi *rhi,
                                            QRhiResourceUpdateBatch *resourceUpdates, QRhiTexture **textures);
 
+struct Q_MULTIMEDIA_EXPORT SubtitleLayout
+{
+    QSize videoSize;
+    QRectF bounds;
+    QTextLayout layout;
+
+    void updateFromVideoFrame(const QVideoFrame &frame);
+    void draw(QPainter *painter, const QRectF &videoRect) const;
+    QImage toImage() const;
+};
+
 }
 
 QT_END_NAMESPACE
diff --git a/src/multimedia/video/qvideowindow.cpp b/src/multimedia/video/qvideowindow.cpp
index 72112d8fa..9b0bb7fc1 100644
--- a/src/multimedia/video/qvideowindow.cpp
+++ b/src/multimedia/video/qvideowindow.cpp
@@ -180,19 +180,21 @@ void QVideoWindowPrivate::initRhi()
     m_textureSampler->create();
 
     m_shaderResourceBindings.reset(m_rhi->newShaderResourceBindings());
+    m_subtitleResourceBindings.reset(m_rhi->newShaderResourceBindings());
+
+    m_subtitleUniformBuf.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, 64 + 64 + 4 + 4));
+    m_subtitleUniformBuf->create();
 }
 
-void QVideoWindowPrivate::updateGraphicsPipeline()
+void QVideoWindowPrivate::setupGraphicsPipeline(QRhiGraphicsPipeline *pipeline, QRhiShaderResourceBindings *bindings, QVideoFrameFormat::PixelFormat fmt)
 {
-    if (!m_graphicsPipeline)
-        m_graphicsPipeline.reset(m_rhi->newGraphicsPipeline());
 
-    m_graphicsPipeline->setTopology(QRhiGraphicsPipeline::TriangleStrip);
-    QShader vs = getShader(QVideoTextureHelper::vertexShaderFileName(format));
+    pipeline->setTopology(QRhiGraphicsPipeline::TriangleStrip);
+    QShader vs = getShader(QVideoTextureHelper::vertexShaderFileName(fmt));
     Q_ASSERT(vs.isValid());
-    QShader fs = getShader(QVideoTextureHelper::fragmentShaderFileName(format));
+    QShader fs = getShader(QVideoTextureHelper::fragmentShaderFileName(fmt));
     Q_ASSERT(fs.isValid());
-    m_graphicsPipeline->setShaderStages({
+    pipeline->setShaderStages({
         { QRhiShaderStage::Vertex, vs },
         { QRhiShaderStage::Fragment, fs }
     });
@@ -204,10 +206,10 @@ void QVideoWindowPrivate::updateGraphicsPipeline()
         { 0, 0, QRhiVertexInputAttribute::Float2, 0 },
         { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) }
     });
-    m_graphicsPipeline->setVertexInputLayout(inputLayout);
-    m_graphicsPipeline->setShaderResourceBindings(m_shaderResourceBindings.get());
-    m_graphicsPipeline->setRenderPassDescriptor(m_renderPass.get());
-    m_graphicsPipeline->create();
+    pipeline->setVertexInputLayout(inputLayout);
+    pipeline->setShaderResourceBindings(bindings);
+    pipeline->setRenderPassDescriptor(m_renderPass.get());
+    pipeline->create();
 }
 
 void QVideoWindowPrivate::updateTextures(QRhiResourceUpdateBatch *rub)
@@ -247,7 +249,46 @@ void QVideoWindowPrivate::updateTextures(QRhiResourceUpdateBatch *rub)
 
     if (fmt != format) {
         format = fmt;
-        updateGraphicsPipeline();
+        if (!m_graphicsPipeline)
+            m_graphicsPipeline.reset(m_rhi->newGraphicsPipeline());
+
+        setupGraphicsPipeline(m_graphicsPipeline.get(), m_shaderResourceBindings.get(), format);
+    }
+}
+
+void QVideoWindowPrivate::updateSubtitle(QRhiResourceUpdateBatch *rub)
+{
+    m_subtitleDirty = false;
+    m_hasSubtitle = !m_currentFrame.subtitleText().isEmpty();
+    if (!m_hasSubtitle)
+        return;
+
+    m_subtitleLayout.updateFromVideoFrame(m_currentFrame);
+    QSize size = m_subtitleLayout.bounds.size().toSize();
+
+    QImage img = m_subtitleLayout.toImage();
+
+    m_subtitleTexture.reset(m_rhi->newTexture(QRhiTexture::RGBA8, size));
+    m_subtitleTexture->create();
+    rub->uploadTexture(m_subtitleTexture.get(), img);
+
+    QRhiShaderResourceBinding bindings[2];
+
+    bindings[0] = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage,
+                                                           m_subtitleUniformBuf.get());
+
+    bindings[1] = QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::FragmentStage,
+                                                            m_subtitleTexture.get(), m_textureSampler.get());
+    m_subtitleResourceBindings->setBindings(bindings, bindings + 2);
+    m_subtitleResourceBindings->create();
+
+    if (!m_subtitlePipeline) {
+        m_subtitlePipeline.reset(m_rhi->newGraphicsPipeline());
+
+        QRhiGraphicsPipeline::TargetBlend blend;
+        blend.enable = true;
+        m_subtitlePipeline->setTargetBlends({ blend });
+        setupGraphicsPipeline(m_subtitlePipeline.get(), m_subtitleResourceBindings.get(), QVideoFrameFormat::Format_RGBA8888);
     }
 }
 
@@ -349,6 +390,9 @@ void QVideoWindowPrivate::render()
     if (m_texturesDirty)
         updateTextures(rub);
 
+    if (m_subtitleDirty)
+        updateSubtitle(rub);
+
     float xscale = 1.f - float(rect.width() - videoRect.width())/float(rect.width());
     float yscale = -1.f + float(rect.height() - videoRect.height())/float(rect.height());
 
@@ -363,6 +407,24 @@ void QVideoWindowPrivate::render()
     QVideoTextureHelper::updateUniformData(&uniformData, m_currentFrame.surfaceFormat(), m_currentFrame, transform, 1.f);
     rub->updateDynamicBuffer(m_uniformBuf.get(), 0, uniformData.size(), uniformData.constData());
 
+    if (m_hasSubtitle) {
+        QMatrix4x4 t = {
+            xscale, 0, 0, 0,
+            0, yscale, 0, 0,
+            0, 0, 1.f, 0,
+            0, 0, 0, 1.f
+        };
+        QSizeF frameSize = m_currentFrame.size();
+        t.translate(0, 2.*m_subtitleLayout.bounds.center().y()/frameSize.height() - 1.);
+        t.scale(m_subtitleLayout.bounds.width()/frameSize.width(),
+                m_subtitleLayout.bounds.height()/frameSize.height());
+
+        QByteArray uniformData(64 + 64 + 4 + 4, Qt::Uninitialized);
+        QVideoFrameFormat fmt(m_subtitleLayout.bounds.size().toSize(), QVideoFrameFormat::Format_ARGB8888);
+        QVideoTextureHelper::updateUniformData(&uniformData, fmt, QVideoFrame(), t, 1.f);
+        rub->updateDynamicBuffer(m_subtitleUniformBuf.get(), 0, uniformData.size(), uniformData.constData());
+    }
+
     QRhiCommandBuffer *cb = m_swapChain->currentFrameCommandBuffer();
     cb->beginPass(m_swapChain->currentFrameRenderTarget(), Qt::black, { 1.0f, 0 }, rub);
     cb->setGraphicsPipeline(m_graphicsPipeline.get());
@@ -374,6 +436,14 @@ void QVideoWindowPrivate::render()
     cb->setVertexInput(0, 1, &vbufBinding);
     cb->draw(4);
 
+    if (m_hasSubtitle) {
+        cb->setGraphicsPipeline(m_subtitlePipeline.get());
+        cb->setShaderResources(m_subtitleResourceBindings.get());
+        const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuf.get(), 0);
+        cb->setVertexInput(0, 1, &vbufBinding);
+        cb->draw(4);
+    }
+
     cb->endPass();
 
     m_rhi->endFrame(m_swapChain.get());
@@ -452,6 +522,8 @@ void QVideoWindow::resizeEvent(QResizeEvent *resizeEvent)
 
 void QVideoWindow::newVideoFrame(const QVideoFrame &frame)
 {
+    if (d->m_currentFrame.subtitleText() != frame.subtitleText())
+        d->m_subtitleDirty = true;
     d->m_currentFrame = frame;
     d->m_texturesDirty = true;
     if (d->isExposed)
diff --git a/src/multimedia/video/qvideowindow_p.h b/src/multimedia/video/qvideowindow_p.h
index 70212d1c2..f95f5ca7f 100644
--- a/src/multimedia/video/qvideowindow_p.h
+++ b/src/multimedia/video/qvideowindow_p.h
@@ -52,6 +52,7 @@
 //
 
 #include <QWindow>
+#include <qtextlayout.h>
 
 #include <QtGui/private/qrhinull_p.h>
 #if QT_CONFIG(opengl)
@@ -94,9 +95,11 @@ public:
     void releaseSwapChain();
 
     void updateTextures(QRhiResourceUpdateBatch *rub);
-    void updateGraphicsPipeline();
+    void updateSubtitle(QRhiResourceUpdateBatch *rub);
     void freeTextures();
 
+    void setupGraphicsPipeline(QRhiGraphicsPipeline *pipeline, QRhiShaderResourceBindings *bindings, QVideoFrameFormat::PixelFormat fmt);
+
     QVideoWindow *q = nullptr;
     Qt::AspectRatioMode aspectRatioMode = Qt::KeepAspectRatio;
 
@@ -117,16 +120,24 @@ public:
     std::unique_ptr<QRhiShaderResourceBindings> m_shaderResourceBindings;
     std::unique_ptr<QRhiGraphicsPipeline> m_graphicsPipeline;
 
+    std::unique_ptr<QRhiTexture> m_subtitleTexture;
+    std::unique_ptr<QRhiShaderResourceBindings> m_subtitleResourceBindings;
+    std::unique_ptr<QRhiGraphicsPipeline> m_subtitlePipeline;
+    std::unique_ptr<QRhiBuffer> m_subtitleUniformBuf;
+
     std::unique_ptr<QVideoSink> m_sink;
     QRhi::Implementation m_graphicsApi = QRhi::Null;
     QSize m_frameSize = QSize(-1, -1);
     QVideoFrame m_currentFrame;
+    QVideoTextureHelper::SubtitleLayout m_subtitleLayout;
 
     bool initialized = false;
     bool isExposed = false;
     bool m_useRhi = true;
     bool m_hasSwapChain = false;
     bool m_texturesDirty = true;
+    bool m_subtitleDirty = false;
+    bool m_hasSubtitle = false;
     QVideoFrameFormat::PixelFormat format = QVideoFrameFormat::Format_Invalid;
 };