Gstreamer: Introduce support of OpenGL plugin

Since uploading to gl texture is quite performance-wise operation
introduced integration of opengl plugin which can provide a way
to avoid uploading to texture on each draw on qt's side.
Which potentially fixes some performance issues especially on embedded devices.

Added glupload and glcolorconvert gst elements to pipeline.
For qml apps current opengl context is provided to the gst plugin to share opengl data with.
It allows at the end to just use gl textures that created and filled inside the gst.

For overlay apps, glimagesink element is added.

Requires gstreamer-gl-1.0 to be installed.
Since this plugin is almost always available, so enabling it by default will break behavior of current pipeline
and might introduce some side effects.
Thus also provided QT_GSTREAMER_USE_OPENGL_PLUGIN env var to explicitly enable OpenGL features from gst.

After this change video frames will be texture based.
But it is still possible to map frames to download their data.

In case if the video surface does not support gl textures,
glmemory will be anyway requested, which will lead to uploading data, downloading
and uploading it again to textures in scene graph video node.

Task-number: QTBUG-66162
Change-Id: I32044ba0bf0c0cf90434d72f8991ad00927e1380
Reviewed-by: Christian Strømme <christian.stromme@qt.io>

8 files changed, 233 insertions, 4 deletions

diff --git a/src/gsttools/gsttools.pro b/src/gsttools/gsttools.pro
index d161fff85..fff039b3b 100644
--- a/src/gsttools/gsttools.pro
+++ b/src/gsttools/gsttools.pro
@@ -82,6 +82,8 @@ qtConfig(gstreamer_0_10) {
         qgstvideorenderersink.cpp
 }
 
+qtConfig(gstreamer_gl): QMAKE_USE += gstreamer_gl
+
 qtConfig(gstreamer_app) {
     QMAKE_USE += gstreamer_app
     PRIVATE_HEADERS += qgstappsrc_p.h
diff --git a/src/gsttools/qgstreamerplayersession.cpp b/src/gsttools/qgstreamerplayersession.cpp
index 9858f61c9..2f6923274 100644
--- a/src/gsttools/qgstreamerplayersession.cpp
+++ b/src/gsttools/qgstreamerplayersession.cpp
@@ -206,13 +206,26 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
     m_videoOutputBin = gst_bin_new("video-output-bin");
     // might not get a parent, take ownership to avoid leak
     qt_gst_object_ref_sink(GST_OBJECT(m_videoOutputBin));
+
+    GstElement *videoOutputSink = m_videoIdentity;
+#if QT_CONFIG(gstreamer_gl)
+    if (QGstUtils::useOpenGL()) {
+        videoOutputSink = gst_element_factory_make("glupload", NULL);
+        GstElement *colorConvert = gst_element_factory_make("glcolorconvert", NULL);
+        gst_bin_add_many(GST_BIN(m_videoOutputBin), videoOutputSink, colorConvert, m_videoIdentity, m_nullVideoSink, NULL);
+        gst_element_link_many(videoOutputSink, colorConvert, m_videoIdentity, NULL);
+    } else {
+        gst_bin_add_many(GST_BIN(m_videoOutputBin), m_videoIdentity, m_nullVideoSink, NULL);
+    }
+#else
     gst_bin_add_many(GST_BIN(m_videoOutputBin), m_videoIdentity, m_nullVideoSink, NULL);
+#endif
     gst_element_link(m_videoIdentity, m_nullVideoSink);
 
     m_videoSink = m_nullVideoSink;
 
     // add ghostpads
-    GstPad *pad = gst_element_get_static_pad(m_videoIdentity,"sink");
+    GstPad *pad = gst_element_get_static_pad(videoOutputSink, "sink");
     gst_element_add_pad(GST_ELEMENT(m_videoOutputBin), gst_ghost_pad_new("sink", pad));
     gst_object_unref(GST_OBJECT(pad));
 
diff --git a/src/gsttools/qgstreamervideooverlay.cpp b/src/gsttools/qgstreamervideooverlay.cpp
index 1f3e28549..6b862e475 100644
--- a/src/gsttools/qgstreamervideooverlay.cpp
+++ b/src/gsttools/qgstreamervideooverlay.cpp
@@ -48,6 +48,8 @@
 #include <gst/video/videooverlay.h>
 #endif
 
+#include <QtMultimedia/private/qtmultimediaglobal_p.h>
+
 QT_BEGIN_NAMESPACE
 
 struct ElementMap
@@ -59,6 +61,9 @@ struct ElementMap
 // Ordered by descending priority
 static const ElementMap elementMap[] =
 {
+#if QT_CONFIG(gstreamer_gl)
+    { "xcb", "glimagesink" },
+#endif
     { "xcb", "vaapisink" },
     { "xcb", "xvimagesink" },
     { "xcb", "ximagesink" }
@@ -340,6 +345,10 @@ static GstElement *findBestVideoSink()
 
     // First, try some known video sinks, depending on the Qt platform plugin in use.
     for (quint32 i = 0; i < (sizeof(elementMap) / sizeof(ElementMap)); ++i) {
+#if QT_CONFIG(gstreamer_gl)
+        if (!QGstUtils::useOpenGL() && qstrcmp(elementMap[i].gstreamerElement, "glimagesink") == 0)
+            continue;
+#endif
         if (platform == QLatin1String(elementMap[i].qtPlatform)
                 && (choice = gst_element_factory_make(elementMap[i].gstreamerElement, NULL))) {
 
diff --git a/src/gsttools/qgstreamervideowidget.cpp b/src/gsttools/qgstreamervideowidget.cpp
index bb85e1eb7..288a9c9c0 100644
--- a/src/gsttools/qgstreamervideowidget.cpp
+++ b/src/gsttools/qgstreamervideowidget.cpp
@@ -173,6 +173,8 @@ bool QGstreamerVideoWidgetControl::eventFilter(QObject *object, QEvent *e)
         }
 
         if (e->type() == QEvent::Paint) {
+            // Update overlay by new size if any.
+            m_videoOverlay.setRenderRectangle(QRect(0, 0, m_widget->width(), m_widget->height()));
             if (m_videoOverlay.isActive())
                 m_videoOverlay.expose(); // triggers a repaint of the last frame
             else
diff --git a/src/gsttools/qgstutils.cpp b/src/gsttools/qgstutils.cpp
index 258cbb404..82af0f1e2 100644
--- a/src/gsttools/qgstutils.cpp
+++ b/src/gsttools/qgstutils.cpp
@@ -1045,11 +1045,13 @@ static const VideoFormat qt_videoFormatLookup[] =
     { QVideoFrame::Format_RGB32 ,  GST_VIDEO_FORMAT_BGRx },
     { QVideoFrame::Format_BGR32 ,  GST_VIDEO_FORMAT_RGBx },
     { QVideoFrame::Format_ARGB32,  GST_VIDEO_FORMAT_BGRA },
+    { QVideoFrame::Format_ABGR32,  GST_VIDEO_FORMAT_RGBA },
     { QVideoFrame::Format_BGRA32,  GST_VIDEO_FORMAT_ARGB },
 #else
     { QVideoFrame::Format_RGB32 ,  GST_VIDEO_FORMAT_xRGB },
     { QVideoFrame::Format_BGR32 ,  GST_VIDEO_FORMAT_xBGR },
     { QVideoFrame::Format_ARGB32,  GST_VIDEO_FORMAT_ARGB },
+    { QVideoFrame::Format_ABGR32,  GST_VIDEO_FORMAT_ABGR },
     { QVideoFrame::Format_BGRA32,  GST_VIDEO_FORMAT_BGRA },
 #endif
     { QVideoFrame::Format_RGB24 ,  GST_VIDEO_FORMAT_RGB },
@@ -1568,6 +1570,12 @@ QVariant QGstUtils::toGStreamerOrientation(const QVariant &value)
 }
 #endif
 
+bool QGstUtils::useOpenGL()
+{
+    static bool result = qEnvironmentVariableIntValue("QT_GSTREAMER_USE_OPENGL_PLUGIN");
+    return result;
+}
+
 void qt_gst_object_ref_sink(gpointer object)
 {
 #if GST_CHECK_VERSION(0,10,24)
diff --git a/src/gsttools/qgstutils_p.h b/src/gsttools/qgstutils_p.h
index 387a2e27a..5a2feec17 100644
--- a/src/gsttools/qgstutils_p.h
+++ b/src/gsttools/qgstutils_p.h
@@ -153,6 +153,8 @@ namespace QGstUtils {
     Q_GSTTOOLS_EXPORT QVariant fromGStreamerOrientation(const QVariant &value);
     Q_GSTTOOLS_EXPORT QVariant toGStreamerOrientation(const QVariant &value);
 #endif
+
+    Q_GSTTOOLS_EXPORT bool useOpenGL();
 }
 
 Q_GSTTOOLS_EXPORT void qt_gst_object_ref_sink(gpointer object);
diff --git a/src/gsttools/qgstvideorenderersink.cpp b/src/gsttools/qgstvideorenderersink.cpp
index 09fdd42a6..49016b952 100644
--- a/src/gsttools/qgstvideorenderersink.cpp
+++ b/src/gsttools/qgstvideorenderersink.cpp
@@ -54,12 +54,32 @@
 
 #include "qgstutils_p.h"
 
+#if QT_CONFIG(gstreamer_gl)
+#include <QOpenGLContext>
+#include <QGuiApplication>
+#include <QWindow>
+#include <qpa/qplatformnativeinterface.h>
+
+#include <gst/gl/gstglconfig.h>
+
+#if GST_GL_HAVE_WINDOW_X11
+#    include <gst/gl/x11/gstgldisplay_x11.h>
+#endif
+#if GST_GL_HAVE_PLATFORM_EGL
+#    include <gst/gl/egl/gstgldisplay_egl.h>
+#endif
+#if GST_CHECK_VERSION(1,11,1) && GST_GL_HAVE_WINDOW_WAYLAND
+#    include <gst/gl/wayland/gstgldisplay_wayland.h>
+#endif
+#endif // #if QT_CONFIG(gstreamer_gl)
+
 //#define DEBUG_VIDEO_SURFACE_SINK
 
 QT_BEGIN_NAMESPACE
 
 QGstDefaultVideoRenderer::QGstDefaultVideoRenderer()
     : m_flushed(true)
+    , m_handleType(QAbstractVideoBuffer::NoHandle)
 {
 }
 
@@ -69,13 +89,33 @@ QGstDefaultVideoRenderer::~QGstDefaultVideoRenderer()
 
 GstCaps *QGstDefaultVideoRenderer::getCaps(QAbstractVideoSurface *surface)
 {
-    return QGstUtils::capsForFormats(surface->supportedPixelFormats());
+#if QT_CONFIG(gstreamer_gl)
+    if (QGstUtils::useOpenGL()) {
+        m_handleType = QAbstractVideoBuffer::GLTextureHandle;
+        auto formats = surface->supportedPixelFormats(m_handleType);
+        // Even if the surface does not support gl textures,
+        // glupload will be added to the pipeline and GLMemory will be requested.
+        // This will lead to upload data to gl textures
+        // and download it when the buffer will be used within rendering.
+        if (formats.isEmpty()) {
+            m_handleType = QAbstractVideoBuffer::NoHandle;
+            formats = surface->supportedPixelFormats(m_handleType);
+        }
+
+        GstCaps *caps = QGstUtils::capsForFormats(formats);
+        for (guint i = 0; i < gst_caps_get_size(caps); ++i)
+            gst_caps_set_features(caps, i, gst_caps_features_from_string("memory:GLMemory"));
+
+        return caps;
+    }
+#endif
+    return QGstUtils::capsForFormats(surface->supportedPixelFormats(QAbstractVideoBuffer::NoHandle));
 }
 
 bool QGstDefaultVideoRenderer::start(QAbstractVideoSurface *surface, GstCaps *caps)
 {
     m_flushed = true;
-    m_format = QGstUtils::formatForCaps(caps, &m_videoInfo);
+    m_format = QGstUtils::formatForCaps(caps, &m_videoInfo, m_handleType);
 
     return m_format.isValid() && surface->start(m_format);
 }
@@ -90,8 +130,21 @@ void QGstDefaultVideoRenderer::stop(QAbstractVideoSurface *surface)
 bool QGstDefaultVideoRenderer::present(QAbstractVideoSurface *surface, GstBuffer *buffer)
 {
     m_flushed = false;
+
+    QGstVideoBuffer *videoBuffer = nullptr;
+#if QT_CONFIG(gstreamer_gl)
+    if (m_format.handleType() == QAbstractVideoBuffer::GLTextureHandle) {
+        GstGLMemory *glmem = GST_GL_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
+        guint textureId = gst_gl_memory_get_texture_id(glmem);
+        videoBuffer = new QGstVideoBuffer(buffer, m_videoInfo, m_format.handleType(), textureId);
+    }
+#endif
+
+    if (!videoBuffer)
+        videoBuffer = new QGstVideoBuffer(buffer, m_videoInfo);
+
     QVideoFrame frame(
-                new QGstVideoBuffer(buffer, m_videoInfo),
+                videoBuffer,
                 m_format.frameSize(),
                 m_format.pixelFormat());
     QGstUtils::setFrameTimeStamps(&frame, buffer);
@@ -145,6 +198,10 @@ QVideoSurfaceGstDelegate::~QVideoSurfaceGstDelegate()
         gst_caps_unref(m_surfaceCaps);
     if (m_startCaps)
         gst_caps_unref(m_startCaps);
+#if QT_CONFIG(gstreamer_gl)
+    if (m_gstGLDisplayContext)
+        gst_object_unref(m_gstGLDisplayContext);
+#endif
 }
 
 GstCaps *QVideoSurfaceGstDelegate::caps()
@@ -254,6 +311,118 @@ GstFlowReturn QVideoSurfaceGstDelegate::render(GstBuffer *buffer)
     return m_renderReturn;
 }
 
+#if QT_CONFIG(gstreamer_gl)
+static GstGLContext *gstGLDisplayContext(QAbstractVideoSurface *surface)
+{
+    QOpenGLContext *glContext = qobject_cast<QOpenGLContext*>(surface->property("GLContext").value<QObject*>());
+    // Context is not ready yet.
+    if (!glContext)
+        return nullptr;
+
+    GstGLDisplay *display = nullptr;
+    const QString platform = QGuiApplication::platformName();
+    const char *contextName = "eglcontext";
+    GstGLPlatform glPlatform = GST_GL_PLATFORM_EGL;
+    QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface();
+
+#if GST_GL_HAVE_WINDOW_X11
+    if (platform == QLatin1String("xcb")) {
+        if (QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGL) {
+            contextName = "glxcontext";
+            glPlatform = GST_GL_PLATFORM_GLX;
+        }
+
+        display = (GstGLDisplay *)gst_gl_display_x11_new_with_display(
+            (Display *)pni->nativeResourceForIntegration("display"));
+    }
+#endif
+
+#if GST_GL_HAVE_PLATFORM_EGL
+    if (!display && platform == QLatin1String("eglfs")) {
+        display = (GstGLDisplay *)gst_gl_display_egl_new_with_egl_display(
+            pni->nativeResourceForIntegration("egldisplay"));
+    }
+#endif
+
+#if GST_CHECK_VERSION(1,11,1)
+#if GST_GL_HAVE_WINDOW_WAYLAND
+    if (!display && platform.startsWith(QLatin1String("wayland"))) {
+        const char *displayName = (platform == QLatin1String("wayland"))
+            ? "display" : "egldisplay";
+
+        display = (GstGLDisplay *)gst_gl_display_wayland_new_with_display(
+            (struct wl_display *)pni->nativeResourceForIntegration(displayName));
+    }
+#endif
+#endif
+
+    if (!display) {
+        qWarning() << "Could not create GstGLDisplay";
+        return nullptr;
+    }
+
+    void *nativeContext = pni->nativeResourceForContext(contextName, glContext);
+    if (!nativeContext)
+        qWarning() << "Could not find resource for" << contextName;
+
+    GstGLContext *appContext = gst_gl_context_new_wrapped(display, (guintptr)nativeContext, glPlatform, GST_GL_API_ANY);
+    if (!appContext)
+        qWarning() << "Could not create wrappped context for platform:" << glPlatform;
+
+    GstGLContext *displayContext = nullptr;
+    GError *error = NULL;
+    gst_gl_display_create_context(display, appContext, &displayContext, &error);
+    if (error) {
+        qWarning() << "Could not create display context:" << error->message;
+        g_clear_error(&error);
+    }
+
+    if (appContext)
+        gst_object_unref(appContext);
+
+    gst_object_unref(display);
+
+    return displayContext;
+}
+#endif // #if QT_CONFIG(gstreamer_gl)
+
+bool QVideoSurfaceGstDelegate::query(GstQuery *query)
+{
+#if QT_CONFIG(gstreamer_gl)
+    if (GST_QUERY_TYPE(query) == GST_QUERY_CONTEXT) {
+        const gchar *type;
+        gst_query_parse_context_type(query, &type);
+
+        if (strcmp(type, "gst.gl.local_context") != 0)
+            return false;
+
+        if (!m_gstGLDisplayContext)
+            m_gstGLDisplayContext = gstGLDisplayContext(m_surface);
+
+        // No context yet.
+        if (!m_gstGLDisplayContext)
+            return false;
+
+        GstContext *context = NULL;
+        gst_query_parse_context(query, &context);
+        context = context ? gst_context_copy(context) : gst_context_new(type, FALSE);
+        GstStructure *structure = gst_context_writable_structure(context);
+#if GST_CHECK_VERSION(1,11,1)
+        gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, m_gstGLDisplayContext, NULL);
+#else
+        gst_structure_set(structure, "context", GST_GL_TYPE_CONTEXT, m_gstGLDisplayContext, NULL);
+#endif
+        gst_query_set_context(query, context);
+        gst_context_unref(context);
+
+        return m_gstGLDisplayContext;
+    }
+#else
+    Q_UNUSED(query);
+#endif
+    return false;
+}
+
 bool QVideoSurfaceGstDelegate::event(QEvent *event)
 {
     if (event->type() == QEvent::UpdateRequest) {
@@ -460,6 +629,7 @@ void QGstVideoRendererSink::class_init(gpointer g_class, gpointer class_data)
     base_sink_class->propose_allocation = QGstVideoRendererSink::propose_allocation;
     base_sink_class->stop = QGstVideoRendererSink::stop;
     base_sink_class->unlock = QGstVideoRendererSink::unlock;
+    base_sink_class->query = QGstVideoRendererSink::query;
 
     GstElementClass *element_class = reinterpret_cast<GstElementClass *>(g_class);
     element_class->change_state = QGstVideoRendererSink::change_state;
@@ -602,4 +772,13 @@ GstFlowReturn QGstVideoRendererSink::show_frame(GstVideoSink *base, GstBuffer *b
     return sink->delegate->render(buffer);
 }
 
+gboolean QGstVideoRendererSink::query(GstBaseSink *base, GstQuery *query)
+{
+    VO_SINK(base);
+    if (sink->delegate->query(query))
+        return TRUE;
+
+    return GST_BASE_SINK_CLASS(sink_parent_class)->query(base, query);
+}
+
 QT_END_NAMESPACE
diff --git a/src/gsttools/qgstvideorenderersink_p.h b/src/gsttools/qgstvideorenderersink_p.h
index d2417a7c9..38854291a 100644
--- a/src/gsttools/qgstvideorenderersink_p.h
+++ b/src/gsttools/qgstvideorenderersink_p.h
@@ -51,6 +51,7 @@
 // We mean it.
 //
 
+#include <QtMultimedia/private/qtmultimediaglobal_p.h>
 #include <gst/video/gstvideosink.h>
 #include <gst/video/video.h>
 
@@ -67,6 +68,13 @@
 
 #include "qgstvideorendererplugin_p.h"
 
+#if QT_CONFIG(gstreamer_gl)
+#ifndef GST_USE_UNSTABLE_API
+#define GST_USE_UNSTABLE_API
+#endif
+#include <gst/gl/gl.h>
+#endif
+
 QT_BEGIN_NAMESPACE
 class QAbstractVideoSurface;
 
@@ -89,6 +97,7 @@ private:
     QVideoSurfaceFormat m_format;
     GstVideoInfo m_videoInfo;
     bool m_flushed;
+    QAbstractVideoBuffer::HandleType m_handleType;
 };
 
 class QVideoSurfaceGstDelegate : public QObject
@@ -110,6 +119,7 @@ public:
     GstFlowReturn render(GstBuffer *buffer);
 
     bool event(QEvent *event) override;
+    bool query(GstQuery *query);
 
 private slots:
     bool handleEvent(QMutexLocker *locker);
@@ -132,6 +142,9 @@ private:
     GstCaps *m_surfaceCaps;
     GstCaps *m_startCaps;
     GstBuffer *m_renderBuffer;
+#if QT_CONFIG(gstreamer_gl)
+    GstGLContext *m_gstGLDisplayContext = nullptr;
+#endif
 
     bool m_notified;
     bool m_stop;
@@ -168,6 +181,7 @@ private:
     static gboolean unlock(GstBaseSink *sink);
 
     static GstFlowReturn show_frame(GstVideoSink *sink, GstBuffer *buffer);
+    static gboolean query(GstBaseSink *element, GstQuery *query);
 
 private:
     QVideoSurfaceGstDelegate *delegate;