Renderer: implement using OpenGL textures from a texture id

Change-Id: Ifc65966c50cb6fc8780206852f92bf1d4a6b4d5b
Task-number: QTBUG-69918
Reviewed-by: Paul Lemire <paul.lemire@kdab.com>

4 files changed, 526 insertions, 0 deletions

diff --git a/tests/manual/sharedtexture/main.cpp b/tests/manual/sharedtexture/main.cpp
new file mode 100644
index 000000000..a85f90ee6
--- /dev/null
+++ b/tests/manual/sharedtexture/main.cpp
@@ -0,0 +1,159 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 Klaralvdalens Datakonsult AB (KDAB).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QApplication>
+#include <QPropertyAnimation>
+
+#include <Qt3DCore/QEntity>
+#include <Qt3DCore/QTransform>
+#include <Qt3DCore/QAspectEngine>
+
+#include <Qt3DRender/QCamera>
+#include <Qt3DRender/QCameraLens>
+#include <Qt3DRender/QRenderAspect>
+#include <Qt3DRender/QTexture>
+#include <Qt3DRender/QDirectionalLight>
+
+#include <Qt3DInput/QInputAspect>
+
+#include <Qt3DExtras/QForwardRenderer>
+#include <Qt3DExtras/QDiffuseMapMaterial>
+#include <Qt3DExtras/QCuboidMesh>
+#include <Qt3DExtras/QOrbitCameraController>
+#include <Qt3DExtras/Qt3DWindow>
+
+#include "videoplayer.h"
+
+Qt3DCore::QEntity *createScene(Qt3DExtras::Qt3DWindow *view, Qt3DRender::QAbstractTexture *diffuseTexture)
+{
+    // Root entity
+    Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity;
+
+    // Material
+    Qt3DExtras::QDiffuseMapMaterial *material = new Qt3DExtras::QDiffuseMapMaterial(rootEntity);
+    material->setDiffuse(diffuseTexture);
+    material->setAmbient(QColor(30, 30, 30));
+
+    // Sphere
+    Qt3DCore::QEntity *sphereEntity = new Qt3DCore::QEntity(rootEntity);
+    Qt3DExtras::QCuboidMesh *cuboidMesh = new Qt3DExtras::QCuboidMesh;
+    Qt3DCore::QTransform *transform = new Qt3DCore::QTransform;
+
+    transform->setRotationX(180);
+
+    QPropertyAnimation *cubeRotateTransformAnimation = new QPropertyAnimation(transform);
+    cubeRotateTransformAnimation->setTargetObject(transform);
+    cubeRotateTransformAnimation->setPropertyName("rotationY");
+    cubeRotateTransformAnimation->setStartValue(QVariant::fromValue(0));
+    cubeRotateTransformAnimation->setEndValue(QVariant::fromValue(360));
+    cubeRotateTransformAnimation->setDuration(10000);
+    cubeRotateTransformAnimation->setLoopCount(-1);
+    cubeRotateTransformAnimation->start();
+
+    sphereEntity->addComponent(cuboidMesh);
+    sphereEntity->addComponent(transform);
+    sphereEntity->addComponent(material);
+
+    // Camera
+    Qt3DRender::QCamera *camera = view->camera();
+    camera->lens()->setPerspectiveProjection(45.0f, 16.0f/9.0f, 0.1f, 1000.0f);
+    camera->setPosition(QVector3D(0, 0, -5.0f));
+    camera->setViewCenter(QVector3D(0, 0, 0));
+
+    // For camera controls
+    Qt3DExtras::QOrbitCameraController *camController = new Qt3DExtras::QOrbitCameraController(rootEntity);
+    camController->setLinearSpeed( 50.0f );
+    camController->setLookSpeed( 180.0f );
+    camController->setCamera(camera);
+
+    Qt3DRender::QDirectionalLight *light = new Qt3DRender::QDirectionalLight();
+    light->setIntensity(0.8f);
+    light->setWorldDirection(camera->viewVector());
+    rootEntity->addComponent(light);
+
+    return rootEntity;
+}
+
+int main(int argc, char* argv[])
+{
+    QSurfaceFormat format = QSurfaceFormat::defaultFormat();
+    format.setMajorVersion(4);
+    format.setMinorVersion(5);
+    format.setProfile(QSurfaceFormat::CoreProfile);
+    format.setRenderableType(QSurfaceFormat::OpenGL);
+    QSurfaceFormat::setDefaultFormat(format);
+
+    // Will make Qt3D and QOpenGLWidget share a common context
+    QApplication::setAttribute(Qt::AA_ShareOpenGLContexts);
+
+    QApplication app(argc, argv);
+
+    // Multimedia player
+    TextureWidget textureWidget;
+    VideoPlayerThread *videoPlayer = new VideoPlayerThread(&textureWidget);
+    videoPlayer->start();
+
+    textureWidget.resize(800, 600);
+    textureWidget.show();
+
+    // Texture object that Qt3D uses to access the texture from the video player
+    Qt3DRender::QSharedGLTexture *sharedTexture = new Qt3DRender::QSharedGLTexture();
+
+    QObject::connect(&textureWidget, &TextureWidget::textureIdChanged,
+                     sharedTexture, &Qt3DRender::QSharedGLTexture::setTextureId);
+
+    // Qt3D Scene
+    Qt3DExtras::Qt3DWindow view;
+    Qt3DCore::QEntity *scene = createScene(&view, sharedTexture);
+    view.setRootEntity(scene);
+    view.show();
+
+    return app.exec();
+}
diff --git a/tests/manual/sharedtexture/sharedtexture.pro b/tests/manual/sharedtexture/sharedtexture.pro
new file mode 100644
index 000000000..b41f43d71
--- /dev/null
+++ b/tests/manual/sharedtexture/sharedtexture.pro
@@ -0,0 +1,12 @@
+!include( ../manual.pri ) {
+    error( "Couldn't find the manual.pri file!" )
+}
+
+QT += widgets 3dcore 3drender 3dinput 3dextras multimedia
+
+SOURCES += \
+    videoplayer.cpp \
+    main.cpp
+
+HEADERS += \
+    videoplayer.h
diff --git a/tests/manual/sharedtexture/videoplayer.cpp b/tests/manual/sharedtexture/videoplayer.cpp
new file mode 100644
index 000000000..f970116b5
--- /dev/null
+++ b/tests/manual/sharedtexture/videoplayer.cpp
@@ -0,0 +1,233 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 Klaralvdalens Datakonsult AB (KDAB).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QMutexLocker>
+#include <QtMultimedia/QVideoFrame>
+
+#include "videoplayer.h"
+
+TextureWidget::TextureWidget(QWidget *parent)
+    : QOpenGLWidget(parent)
+    , m_texture(QOpenGLTexture::Target2D)
+{
+    // Lock mutex so that we never process a frame until we have been initialized
+    m_mutex.lock();
+}
+
+// Main thread
+void TextureWidget::initializeGL()
+{
+    initializeOpenGLFunctions();
+    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+
+    if (!m_shader.addShaderFromSourceCode(QOpenGLShader::Vertex,
+                                          "#version 330\n"
+                                          "out vec2 coords;\n"
+                                          "const vec2 positions[6] = vec2[] ("
+                                          "    vec2(-1.0, 1.0),"
+                                          "    vec2(-1.0, -1.0),"
+                                          "    vec2(1.0, 1.0),"
+                                          "    vec2(1.0, 1.0),"
+                                          "    vec2(-1.0, -1.0),"
+                                          "    vec2(1.0, -1.0));\n"
+                                          "const vec2 texCoords[6] = vec2[] ("
+                                          "    vec2(0.0, 0.0),"
+                                          "    vec2(0.0, 1.0),"
+                                          "    vec2(1.0, 0.0),"
+                                          "    vec2(1.0, 0.0),"
+                                          "    vec2(0.0, 1.0),"
+                                          "    vec2(1.0, 1.0));\n"
+                                          "void main() {\n"
+                                          "    coords = texCoords[gl_VertexID];\n"
+                                          "    gl_Position = vec4(positions[gl_VertexID], 0.0, 1.0);\n"
+                                          "}"))
+        qDebug() << "Failed to load vertex shader" << m_shader.log();
+    if (!m_shader.addShaderFromSourceCode(QOpenGLShader::Fragment,
+                                          "#version 330\n"
+                                          "in vec2 coords;\n"
+                                          "uniform sampler2D video_texture;\n"
+                                          "out vec4 fragColor;\n"
+                                          "void main() {\n"
+                                          "    fragColor = texture(video_texture, coords);\n"
+                                          "}"))
+        qDebug() << "Failed to load fragment shader" << m_shader.log();
+    if (!m_shader.link())
+        qDebug() << "Failed to link shaders" << m_shader.log();
+
+    qDebug() << Q_FUNC_INFO << context()->shareContext();
+
+    m_vao.create();
+    // Allow rendering/frame acquisition to go on
+    m_mutex.unlock();
+}
+
+// Main thread
+void TextureWidget::paintGL()
+{
+    QMutexLocker lock(&m_mutex);
+    glViewport(0, 0, width(), height());
+
+    glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT);
+
+    if (!m_texture.isCreated())
+        return;
+
+    m_shader.bind();
+
+    m_texture.bind(0);
+    m_shader.setUniformValue("video_texture", 0);
+
+    m_vao.bind();
+    glDrawArrays(GL_TRIANGLES, 0, 6);
+
+    m_vao.release();
+    m_shader.release();
+}
+
+// Video Player thread
+void TextureWidget::setVideoFrame(const QVideoFrame &frame)
+{
+    // Ensure we won't be rendering while we are processing the frame
+    QMutexLocker lock(&m_mutex);
+
+    QVideoFrame f = frame;
+
+    // Map frame
+    if (!f.map(QAbstractVideoBuffer::ReadOnly))
+        return;
+
+    makeCurrent();
+
+    // Create or recreate texture
+    if (m_texture.width() != f.width() || m_texture.height() != f.height()) {
+        if (m_texture.isCreated())
+            m_texture.destroy();
+
+        m_texture.setSize(f.width(), f.height());
+        m_texture.setFormat(QOpenGLTexture::RGBA32F);
+        m_texture.setWrapMode(QOpenGLTexture::ClampToBorder);
+        m_texture.setMinificationFilter(QOpenGLTexture::Nearest);
+        m_texture.setMagnificationFilter(QOpenGLTexture::Nearest);
+        m_texture.allocateStorage();
+
+        m_texture.create();
+        emit textureIdChanged(m_texture.textureId());
+    }
+
+    const QVideoFrame::PixelFormat pFormat = f.pixelFormat();
+    if (pFormat == QVideoFrame::Format_RGB32) {
+        m_texture.setData(QOpenGLTexture::RGBA, QOpenGLTexture::UInt8, f.bits());
+    }
+
+    doneCurrent();
+
+    // Request display udpate
+    QOpenGLWidget::update();
+
+    // Unmap
+    f.unmap();
+}
+
+QList<QVideoFrame::PixelFormat> GLVideoSurface::supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const
+{
+    if (type == QAbstractVideoBuffer::NoHandle)
+        return {
+            QVideoFrame::Format_RGB32,
+            QVideoFrame::Format_ARGB32,
+            QVideoFrame::Format_BGR32,
+            QVideoFrame::Format_BGRA32
+        };
+    return {};
+}
+
+// Video player thread
+bool GLVideoSurface::present(const QVideoFrame &frame)
+{
+    emit onNewFrame(frame);
+    return true;
+}
+
+VideoPlayerThread::VideoPlayerThread(TextureWidget *textureWidget)
+    : QThread(textureWidget)
+    , m_player(new QMediaPlayer(nullptr, QMediaPlayer::VideoSurface))
+    , m_surface(new GLVideoSurface())
+{
+    m_player->moveToThread(this);
+    m_player->setMedia(QUrl("https://www.sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4"));
+
+    // Tell player to render on GLVideoSurface
+    m_surface->moveToThread(this);
+    m_player->setVideoOutput(m_surface.get());
+
+    // Display errors
+    QObject::connect(m_player.get(), QOverload<QMediaPlayer::Error>::of(&QMediaPlayer::error),
+                     m_player.get(), [this] (QMediaPlayer::Error e) {
+        qDebug() << Q_FUNC_INFO << e << m_player->errorString();
+    });
+
+    // Repeat video indefinitely
+    QObject::connect(m_player.get(), &QMediaPlayer::stateChanged, m_player.get(), [this] (QMediaPlayer::State state) {
+        if (state == QMediaPlayer::StoppedState)
+            m_player->play();
+    });
+
+    // Start playing when thread starts
+    QObject::connect(this, &QThread::started, this, [this] { m_player->play(); });
+
+    // Direct connection between 2 objects living in different threads
+    QObject::connect(m_surface.get(), &GLVideoSurface::onNewFrame,
+                     textureWidget, &TextureWidget::setVideoFrame, Qt::DirectConnection);
+}
+
+VideoPlayerThread::~VideoPlayerThread()
+{
+    exit(0);
+    wait();
+}
diff --git a/tests/manual/sharedtexture/videoplayer.h b/tests/manual/sharedtexture/videoplayer.h
new file mode 100644
index 000000000..377ea57fe
--- /dev/null
+++ b/tests/manual/sharedtexture/videoplayer.h
@@ -0,0 +1,122 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 Klaralvdalens Datakonsult AB (KDAB).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QThread>
+#include <QMutex>
+
+#include <QOpenGLWidget>
+#include <QOpenGLFunctions>
+#include <QOpenGLVertexArrayObject>
+#include <QOpenGLBuffer>
+#include <QOpenGLTexture>
+#include <QOpenGLShaderProgram>
+
+#include <QtMultimedia/QAbstractVideoSurface>
+#include <QtMultimedia/QMediaPlayer>
+
+#include <memory>
+
+class TextureWidget : public QOpenGLWidget, private QOpenGLFunctions
+{
+    Q_OBJECT
+    Q_PROPERTY(int textureId READ textureId NOTIFY textureIdChanged)
+public:
+    TextureWidget(QWidget *parent = nullptr);
+
+    int textureId() { return m_texture.textureId(); }
+
+private:
+    // MainThread
+    void initializeGL() override;
+
+    // Main thread
+    void paintGL() override;
+
+public Q_SLOTS:
+    // Called from Video player thread
+    void setVideoFrame(const QVideoFrame &frame);
+
+Q_SIGNALS:
+    void textureIdChanged(int textureId);
+
+private:
+    QOpenGLVertexArrayObject m_vao;
+    QOpenGLShaderProgram m_shader;
+    QOpenGLTexture m_texture;
+    QMutex m_mutex;
+};
+
+
+class GLVideoSurface : public QAbstractVideoSurface
+{
+    Q_OBJECT
+public:
+    QList<QVideoFrame::PixelFormat> supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const override;
+
+    // Call in VideaPlayerThread context
+    bool present(const QVideoFrame &frame) override;
+
+Q_SIGNALS:
+    void onNewFrame(const QVideoFrame &frame);
+};
+
+
+class VideoPlayerThread : public QThread
+{
+    Q_OBJECT
+public:
+    VideoPlayerThread(TextureWidget *textureWidget);
+    ~VideoPlayerThread();
+
+private:
+    TextureWidget *m_textureWidget;
+    std::unique_ptr<QMediaPlayer> m_player;
+    std::unique_ptr<GLVideoSurface> m_surface;
+};