diff options
5 files changed, 269 insertions, 109 deletions
diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp index 5306fe918..55f71d735 100644 --- a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp +++ b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp @@ -49,22 +49,39 @@ #include <qcoreapplication.h> #include <qopenglcontext.h> #include <qopenglfunctions.h> +#include <qopenglshaderprogram.h> +#include <qopenglframebufferobject.h> QT_BEGIN_NAMESPACE -#define ExternalGLTextureHandle QAbstractVideoBuffer::HandleType(QAbstractVideoBuffer::UserHandle + 1) +static const GLfloat g_vertex_data[] = { + -1.f, 1.f, + 1.f, 1.f, + 1.f, -1.f, + -1.f, -1.f +}; + +static const GLfloat g_texture_data[] = { + 0.f, 0.f, + 1.f, 0.f, + 1.f, 1.f, + 0.f, 1.f +}; -TextureDeleter::~TextureDeleter() +OpenGLResourcesDeleter::~OpenGLResourcesDeleter() { - glDeleteTextures(1, &m_id); + glDeleteTextures(1, &m_textureID); + delete m_fbo; + delete m_program; } class AndroidTextureVideoBuffer : public QAbstractVideoBuffer { public: - AndroidTextureVideoBuffer(JSurfaceTexture *surface) - : QAbstractVideoBuffer(ExternalGLTextureHandle) - , m_surfaceTexture(surface) + AndroidTextureVideoBuffer(QAndroidVideoRendererControl *control) + : QAbstractVideoBuffer(GLTextureHandle) + , m_control(control) + , m_textureUpdated(false) { } @@ -76,18 +93,18 @@ public: QVariant handle() const { - if (m_data.isEmpty()) { + if (!m_textureUpdated) { // update the video texture (called from the render thread) - m_surfaceTexture->updateTexImage(); - m_data << (uint)m_surfaceTexture->textureID() << m_surfaceTexture->getTransformMatrix(); + m_control->renderFrameToFbo(); + m_textureUpdated = true; } - return m_data; + return m_control->m_fbo->texture(); } private: - mutable JSurfaceTexture *m_surfaceTexture; - mutable QVariantList m_data; + mutable QAndroidVideoRendererControl *m_control; + mutable bool m_textureUpdated; }; QAndroidVideoRendererControl::QAndroidVideoRendererControl(QObject *parent) @@ -97,7 +114,9 @@ QAndroidVideoRendererControl::QAndroidVideoRendererControl(QObject *parent) , m_surfaceTexture(0) , m_surfaceHolder(0) , m_externalTex(0) - , m_textureDeleter(0) + , m_fbo(0) + , m_program(0) + , m_glDeleter(0) { } @@ -117,8 +136,8 @@ QAndroidVideoRendererControl::~QAndroidVideoRendererControl() delete m_surfaceHolder; m_surfaceHolder = 0; } - if (m_textureDeleter) - m_textureDeleter->deleteLater(); + if (m_glDeleter) + m_glDeleter->deleteLater(); } QAbstractVideoSurface *QAndroidVideoRendererControl::surface() const @@ -162,7 +181,8 @@ bool QAndroidVideoRendererControl::initSurfaceTexture() // for the GL render thread to call us back to do it. if (QOpenGLContext::currentContext()) { glGenTextures(1, &m_externalTex); - m_textureDeleter = new TextureDeleter(m_externalTex); + m_glDeleter = new OpenGLResourcesDeleter; + m_glDeleter->setTexture(m_externalTex); } else if (!m_externalTex) { return false; } @@ -174,9 +194,9 @@ bool QAndroidVideoRendererControl::initSurfaceTexture() } else { delete m_surfaceTexture; m_surfaceTexture = 0; - m_textureDeleter->deleteLater(); + m_glDeleter->deleteLater(); m_externalTex = 0; - m_textureDeleter = 0; + m_glDeleter = 0; } return m_surfaceTexture != 0; @@ -208,6 +228,8 @@ jobject QAndroidVideoRendererControl::surfaceTexture() void QAndroidVideoRendererControl::setVideoSize(const QSize &size) { + QMutexLocker locker(&m_mutex); + if (m_nativeSize == size) return; @@ -228,7 +250,7 @@ void QAndroidVideoRendererControl::onFrameAvailable() if (!m_nativeSize.isValid() || !m_surface) return; - QAbstractVideoBuffer *buffer = new AndroidTextureVideoBuffer(m_surfaceTexture); + QAbstractVideoBuffer *buffer = new AndroidTextureVideoBuffer(this); QVideoFrame frame(buffer, m_nativeSize, QVideoFrame::Format_BGR32); if (m_surface->isActive() && (m_surface->surfaceFormat().pixelFormat() != frame.pixelFormat() @@ -237,8 +259,8 @@ void QAndroidVideoRendererControl::onFrameAvailable() } if (!m_surface->isActive()) { - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), ExternalGLTextureHandle); - format.setScanLineDirection(QVideoSurfaceFormat::BottomToTop); + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), + QAbstractVideoBuffer::GLTextureHandle); m_surface->start(format); } @@ -247,13 +269,114 @@ void QAndroidVideoRendererControl::onFrameAvailable() m_surface->present(frame); } +void QAndroidVideoRendererControl::renderFrameToFbo() +{ + QMutexLocker locker(&m_mutex); + + createGLResources(); + + m_surfaceTexture->updateTexImage(); + + // save current render states + GLboolean stencilTestEnabled; + GLboolean depthTestEnabled; + GLboolean scissorTestEnabled; + GLboolean blendEnabled; + glGetBooleanv(GL_STENCIL_TEST, &stencilTestEnabled); + glGetBooleanv(GL_DEPTH_TEST, &depthTestEnabled); + glGetBooleanv(GL_SCISSOR_TEST, &scissorTestEnabled); + glGetBooleanv(GL_BLEND, &blendEnabled); + + if (stencilTestEnabled) + glDisable(GL_STENCIL_TEST); + if (depthTestEnabled) + glDisable(GL_DEPTH_TEST); + if (scissorTestEnabled) + glDisable(GL_SCISSOR_TEST); + if (blendEnabled) + glDisable(GL_BLEND); + + m_fbo->bind(); + + glViewport(0, 0, m_nativeSize.width(), m_nativeSize.height()); + + m_program->bind(); + m_program->enableAttributeArray(0); + m_program->enableAttributeArray(1); + m_program->setUniformValue("frameTexture", GLuint(0)); + m_program->setUniformValue("texMatrix", m_surfaceTexture->getTransformMatrix()); + + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, g_vertex_data); + glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, g_texture_data); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + + m_program->disableAttributeArray(0); + m_program->disableAttributeArray(1); + + glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0); + m_fbo->release(); + + // restore render states + if (stencilTestEnabled) + glEnable(GL_STENCIL_TEST); + if (depthTestEnabled) + glEnable(GL_DEPTH_TEST); + if (scissorTestEnabled) + glEnable(GL_SCISSOR_TEST); + if (blendEnabled) + glEnable(GL_BLEND); +} + +void QAndroidVideoRendererControl::createGLResources() +{ + if (!m_fbo || m_fbo->size() != m_nativeSize) { + delete m_fbo; + m_fbo = new QOpenGLFramebufferObject(m_nativeSize); + m_glDeleter->setFbo(m_fbo); + } + + if (!m_program) { + m_program = new QOpenGLShaderProgram; + + QOpenGLShader *vertexShader = new QOpenGLShader(QOpenGLShader::Vertex, m_program); + vertexShader->compileSourceCode("attribute highp vec4 vertexCoordsArray; \n" \ + "attribute highp vec2 textureCoordArray; \n" \ + "uniform highp mat4 texMatrix; \n" \ + "varying highp vec2 textureCoords; \n" \ + "void main(void) \n" \ + "{ \n" \ + " gl_Position = vertexCoordsArray; \n" \ + " textureCoords = (texMatrix * vec4(textureCoordArray, 0.0, 1.0)).xy; \n" \ + "}\n"); + m_program->addShader(vertexShader); + + QOpenGLShader *fragmentShader = new QOpenGLShader(QOpenGLShader::Fragment, m_program); + fragmentShader->compileSourceCode("#extension GL_OES_EGL_image_external : require \n" \ + "varying highp vec2 textureCoords; \n" \ + "uniform samplerExternalOES frameTexture; \n" \ + "void main() \n" \ + "{ \n" \ + " gl_FragColor = texture2D(frameTexture, textureCoords); \n" \ + "}\n"); + m_program->addShader(fragmentShader); + + m_program->bindAttributeLocation("vertexCoordsArray", 0); + m_program->bindAttributeLocation("textureCoordArray", 1); + m_program->link(); + + m_glDeleter->setShaderProgram(m_program); + } +} + void QAndroidVideoRendererControl::customEvent(QEvent *e) { if (e->type() == QEvent::User) { // This is running in the render thread (OpenGL enabled) if (!m_externalTex) { glGenTextures(1, &m_externalTex); - m_textureDeleter = new TextureDeleter(m_externalTex); // will be deleted in the correct thread + m_glDeleter = new OpenGLResourcesDeleter; // will cleanup GL resources in the correct thread + m_glDeleter->setTexture(m_externalTex); emit readyChanged(true); } } diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.h b/src/plugins/android/src/common/qandroidvideorendercontrol.h index 5d9130c07..6ce1e2dd4 100644 --- a/src/plugins/android/src/common/qandroidvideorendercontrol.h +++ b/src/plugins/android/src/common/qandroidvideorendercontrol.h @@ -43,22 +43,37 @@ #define QANDROIDVIDEORENDERCONTROL_H #include <qvideorenderercontrol.h> +#include <qmutex.h> #include "qandroidvideooutput.h" #include "jsurfacetexture.h" QT_BEGIN_NAMESPACE class JSurfaceTextureHolder; +class QOpenGLTexture; +class QOpenGLFramebufferObject; +class QOpenGLShaderProgram; -class TextureDeleter : public QObject +class OpenGLResourcesDeleter : public QObject { Q_OBJECT public: - TextureDeleter(uint id) : m_id(id) { } - ~TextureDeleter(); + OpenGLResourcesDeleter() + : m_textureID(0) + , m_fbo(0) + , m_program(0) + { } + + ~OpenGLResourcesDeleter(); + + void setTexture(quint32 id) { m_textureID = id; } + void setFbo(QOpenGLFramebufferObject *fbo) { m_fbo = fbo; } + void setShaderProgram(QOpenGLShaderProgram *prog) { m_program = prog; } private: - uint m_id; + quint32 m_textureID; + QOpenGLFramebufferObject *m_fbo; + QOpenGLShaderProgram *m_program; }; class QAndroidVideoRendererControl : public QVideoRendererControl, public QAndroidVideoOutput @@ -88,6 +103,10 @@ private Q_SLOTS: private: bool initSurfaceTexture(); + void renderFrameToFbo(); + void createGLResources(); + + QMutex m_mutex; QAbstractVideoSurface *m_surface; QSize m_nativeSize; @@ -95,8 +114,13 @@ private: QJNIObjectPrivate *m_androidSurface; JSurfaceTexture *m_surfaceTexture; JSurfaceTextureHolder *m_surfaceHolder; - uint m_externalTex; - TextureDeleter *m_textureDeleter; + + quint32 m_externalTex; + QOpenGLFramebufferObject *m_fbo; + QOpenGLShaderProgram *m_program; + OpenGLResourcesDeleter *m_glDeleter; + + friend class AndroidTextureVideoBuffer; }; QT_END_NAMESPACE diff --git a/src/plugins/android/videonode/qandroidsgvideonode.cpp b/src/plugins/android/videonode/qandroidsgvideonode.cpp index 7f13dc981..8c441a748 100644 --- a/src/plugins/android/videonode/qandroidsgvideonode.cpp +++ b/src/plugins/android/videonode/qandroidsgvideonode.cpp @@ -61,41 +61,42 @@ public: } protected: + const char *vertexShader() const { - return - "uniform highp mat4 qt_Matrix; \n" - "uniform highp mat4 texMatrix; \n" - "attribute highp vec4 qt_VertexPosition; \n" - "attribute highp vec2 qt_VertexTexCoord; \n" - "varying highp vec2 qt_TexCoord; \n" - "void main() { \n" - " qt_TexCoord = (texMatrix * vec4(qt_VertexTexCoord, 0.0, 1.0)).xy; \n" - " gl_Position = qt_Matrix * qt_VertexPosition; \n" - "}"; + const char *shader = + "uniform highp mat4 qt_Matrix; \n" + "attribute highp vec4 qt_VertexPosition; \n" + "attribute highp vec2 qt_VertexTexCoord; \n" + "varying highp vec2 qt_TexCoord; \n" + "void main() { \n" + " qt_TexCoord = qt_VertexTexCoord; \n" + " gl_Position = qt_Matrix * qt_VertexPosition; \n" + "}"; + return shader; } const char *fragmentShader() const { - return - "#extension GL_OES_EGL_image_external : require \n" - "uniform samplerExternalOES videoTexture; \n" - "uniform lowp float opacity; \n" - "varying highp vec2 qt_TexCoord; \n" - "void main() \n" - "{ \n" - " gl_FragColor = texture2D(videoTexture, qt_TexCoord) * opacity; \n" - "}"; + static const char *shader = + "uniform sampler2D rgbTexture;" + "uniform lowp float opacity;" + "" + "varying highp vec2 qt_TexCoord;" + "" + "void main()" + "{" + " gl_FragColor = texture2D(rgbTexture, qt_TexCoord) * opacity;" + "}"; + return shader; } void initialize() { m_id_matrix = program()->uniformLocation("qt_Matrix"); - m_id_texMatrix = program()->uniformLocation("texMatrix"); - m_id_texture = program()->uniformLocation("videoTexture"); + m_id_Texture = program()->uniformLocation("rgbTexture"); m_id_opacity = program()->uniformLocation("opacity"); } int m_id_matrix; - int m_id_texMatrix; - int m_id_texture; + int m_id_Texture; int m_id_opacity; }; @@ -104,15 +105,12 @@ class QAndroidSGVideoNodeMaterial : public QSGMaterial public: QAndroidSGVideoNodeMaterial() : m_textureId(0) + , m_textureUpdated(false) + , m_opacity(1.0) { setFlag(Blending, false); } - ~QAndroidSGVideoNodeMaterial() - { - m_frame = QVideoFrame(); - } - QSGMaterialType *type() const { static QSGMaterialType theType; return &theType; @@ -124,81 +122,93 @@ public: int compare(const QSGMaterial *other) const { const QAndroidSGVideoNodeMaterial *m = static_cast<const QAndroidSGVideoNodeMaterial *>(other); - return m_textureId - m->m_textureId; - } + int diff = m_textureId - m->m_textureId; + if (diff) + return diff; - void setVideoFrame(const QVideoFrame &frame) { - QMutexLocker lock(&m_frameMutex); - m_frame = frame; + return (m_opacity > m->m_opacity) ? 1 : -1; } - bool updateTexture() - { - QMutexLocker lock(&m_frameMutex); - bool texMatrixDirty = false; - - if (m_frame.isValid()) { - QVariantList list = m_frame.handle().toList(); - - GLuint texId = list.at(0).toUInt(); - QMatrix4x4 mat = qvariant_cast<QMatrix4x4>(list.at(1)); - - texMatrixDirty = texId != m_textureId || mat != m_texMatrix; - - m_textureId = texId; - m_texMatrix = mat; - - // the texture is already bound and initialized at this point, - // no need to call glTexParams + void updateBlending() { + setFlag(Blending, qFuzzyCompare(m_opacity, qreal(1.0)) ? false : true); + } - } else { - m_textureId = 0; + void updateTexture(GLuint id, const QSize &size) { + if (m_textureId != id || m_textureSize != size) { + m_textureId = id; + m_textureSize = size; + m_textureUpdated = true; } + } - return texMatrixDirty; + void bind() + { + glBindTexture(GL_TEXTURE_2D, m_textureId); + if (m_textureUpdated) { + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + m_textureUpdated = false; + } } - QVideoFrame m_frame; - QMutex m_frameMutex; + QSize m_textureSize; GLuint m_textureId; - QMatrix4x4 m_texMatrix; + bool m_textureUpdated; + qreal m_opacity; }; + +QAndroidSGVideoNode::QAndroidSGVideoNode(const QVideoSurfaceFormat &format) + : m_format(format) +{ + setFlags(OwnsMaterial | UsePreprocess); + m_material = new QAndroidSGVideoNodeMaterial; + setMaterial(m_material); +} + +QAndroidSGVideoNode::~QAndroidSGVideoNode() +{ + m_frame = QVideoFrame(); +} + +void QAndroidSGVideoNode::setCurrentFrame(const QVideoFrame &frame) +{ + QMutexLocker lock(&m_frameMutex); + m_frame = frame; + markDirty(DirtyMaterial); +} + void QAndroidSGVideoNodeMaterialShader::updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) { Q_UNUSED(oldMaterial); QAndroidSGVideoNodeMaterial *mat = static_cast<QAndroidSGVideoNodeMaterial *>(newMaterial); - program()->setUniformValue(m_id_texture, 0); + program()->setUniformValue(m_id_Texture, 0); - if (mat->updateTexture()) - program()->setUniformValue(m_id_texMatrix, mat->m_texMatrix); + mat->bind(); - if (state.isOpacityDirty()) - program()->setUniformValue(m_id_opacity, state.opacity()); + if (state.isOpacityDirty()) { + mat->m_opacity = state.opacity(); + mat->updateBlending(); + program()->setUniformValue(m_id_opacity, GLfloat(mat->m_opacity)); + } if (state.isMatrixDirty()) program()->setUniformValue(m_id_matrix, state.combinedMatrix()); } -QAndroidSGVideoNode::QAndroidSGVideoNode(const QVideoSurfaceFormat &format) - : m_format(format) +void QAndroidSGVideoNode::preprocess() { - setFlag(QSGNode::OwnsMaterial); - m_material = new QAndroidSGVideoNodeMaterial; - setMaterial(m_material); -} + QMutexLocker lock(&m_frameMutex); -void QAndroidSGVideoNode::setCurrentFrame(const QVideoFrame &frame) -{ - m_material->setVideoFrame(frame); - markDirty(DirtyMaterial); -} + GLuint texId = 0; + if (m_frame.isValid()) + texId = m_frame.handle().toUInt(); -QVideoFrame::PixelFormat QAndroidSGVideoNode::pixelFormat() const -{ - return m_format.pixelFormat(); + m_material->updateTexture(texId, m_frame.size()); } QT_END_NAMESPACE diff --git a/src/plugins/android/videonode/qandroidsgvideonode.h b/src/plugins/android/videonode/qandroidsgvideonode.h index b5b383fb6..5da72dd2d 100644 --- a/src/plugins/android/videonode/qandroidsgvideonode.h +++ b/src/plugins/android/videonode/qandroidsgvideonode.h @@ -43,6 +43,7 @@ #define QANDROIDSGVIDEONODE_H #include <private/qsgvideonode_p.h> +#include <qmutex.h> QT_BEGIN_NAMESPACE @@ -52,14 +53,18 @@ class QAndroidSGVideoNode : public QSGVideoNode { public: QAndroidSGVideoNode(const QVideoSurfaceFormat &format); + ~QAndroidSGVideoNode(); void setCurrentFrame(const QVideoFrame &frame); - QVideoFrame::PixelFormat pixelFormat() const; + QVideoFrame::PixelFormat pixelFormat() const { return m_format.pixelFormat(); } + + void preprocess(); private: - QVideoSurfaceFormat m_format; QAndroidSGVideoNodeMaterial *m_material; + QMutex m_frameMutex; QVideoFrame m_frame; + QVideoSurfaceFormat m_format; }; QT_END_NAMESPACE diff --git a/src/plugins/android/videonode/qandroidsgvideonodeplugin.cpp b/src/plugins/android/videonode/qandroidsgvideonodeplugin.cpp index 155c66ada..e1fb286a2 100644 --- a/src/plugins/android/videonode/qandroidsgvideonodeplugin.cpp +++ b/src/plugins/android/videonode/qandroidsgvideonodeplugin.cpp @@ -44,14 +44,12 @@ QT_BEGIN_NAMESPACE -#define ExternalGLTextureHandle (QAbstractVideoBuffer::UserHandle + 1) - QList<QVideoFrame::PixelFormat> QAndroidSGVideoNodeFactoryPlugin::supportedPixelFormats( QAbstractVideoBuffer::HandleType handleType) const { QList<QVideoFrame::PixelFormat> pixelFormats; - if (handleType == ExternalGLTextureHandle) + if (handleType == QAbstractVideoBuffer::GLTextureHandle) pixelFormats.append(QVideoFrame::Format_BGR32); return pixelFormats; |