Merge remote-tracking branch 'origin/dev' into wip/cmake

Take 5.

Change-Id: Ifb2d20e95ba824e45e667fba6c2ba45389991cc3

6 files changed, 288 insertions, 5 deletions

diff --git a/src/gui/util/qshaderformat.cpp b/src/gui/util/qshaderformat.cpp
index 324d84ffe1..ea86dd6ca5 100644
--- a/src/gui/util/qshaderformat.cpp
+++ b/src/gui/util/qshaderformat.cpp
@@ -43,6 +43,7 @@ QT_BEGIN_NAMESPACE
 
 QShaderFormat::QShaderFormat() noexcept
     : m_api(NoApi)
+    , m_shaderType(Fragment)
 {
 }
 
@@ -106,6 +107,9 @@ bool QShaderFormat::supports(const QShaderFormat &other) const noexcept
     if (m_version < other.m_version)
         return false;
 
+    if (m_shaderType != other.m_shaderType)
+        return false;
+
     const auto containsAllExtensionsFromOther = std::includes(m_extensions.constBegin(),
                                                               m_extensions.constEnd(),
                                                               other.m_extensions.constBegin(),
@@ -119,12 +123,23 @@ bool QShaderFormat::supports(const QShaderFormat &other) const noexcept
     return true;
 }
 
+QShaderFormat::ShaderType QShaderFormat::shaderType() const Q_DECL_NOTHROW
+{
+    return m_shaderType;
+}
+
+void QShaderFormat::setShaderType(QShaderFormat::ShaderType shaderType) Q_DECL_NOTHROW
+{
+    m_shaderType = shaderType;
+}
+
 bool operator==(const QShaderFormat &lhs, const QShaderFormat &rhs) noexcept
 {
     return lhs.api() == rhs.api()
         && lhs.version() == rhs.version()
         && lhs.extensions() == rhs.extensions()
-        && lhs.vendor() == rhs.vendor();
+        && lhs.vendor() == rhs.vendor()
+        && lhs.shaderType() == rhs.shaderType();
 }
 
 QT_END_NAMESPACE
diff --git a/src/gui/util/qshaderformat_p.h b/src/gui/util/qshaderformat_p.h
index 84bf71fdb6..c50d7d5729 100644
--- a/src/gui/util/qshaderformat_p.h
+++ b/src/gui/util/qshaderformat_p.h
@@ -69,6 +69,15 @@ public:
         OpenGLES
     };
 
+    enum ShaderType : int {
+        Vertex = 0,
+        TessellationControl,
+        TessellationEvaluation,
+        Geometry,
+        Fragment,
+        Compute
+    };
+
     Q_GUI_EXPORT QShaderFormat() noexcept;
 
     Q_GUI_EXPORT Api api() const noexcept;
@@ -86,11 +95,15 @@ public:
     Q_GUI_EXPORT bool isValid() const noexcept;
     Q_GUI_EXPORT bool supports(const QShaderFormat &other) const noexcept;
 
+    Q_GUI_EXPORT ShaderType shaderType() const Q_DECL_NOTHROW;
+    Q_GUI_EXPORT void setShaderType(ShaderType shaderType) Q_DECL_NOTHROW;
+
 private:
     Api m_api;
     QVersionNumber m_version;
     QStringList m_extensions;
     QString m_vendor;
+    ShaderType m_shaderType;
 };
 
 Q_GUI_EXPORT bool operator==(const QShaderFormat &lhs, const QShaderFormat &rhs) noexcept;
diff --git a/src/gui/util/qshadergenerator.cpp b/src/gui/util/qshadergenerator.cpp
index 60cf5a2fc5..bcb985de54 100644
--- a/src/gui/util/qshadergenerator.cpp
+++ b/src/gui/util/qshadergenerator.cpp
@@ -40,9 +40,12 @@
 #include "qshadergenerator_p.h"
 
 #include "qshaderlanguage_p.h"
+#include <QRegularExpression>
 
 QT_BEGIN_NAMESPACE
 
+Q_LOGGING_CATEGORY(ShaderGenerator, "ShaderGenerator", QtWarningMsg)
+
 namespace
 {
     QByteArray toGlsl(QShaderLanguage::StorageQualifier qualifier, const QShaderFormat &format)
@@ -56,7 +59,10 @@ namespace
             case QShaderLanguage::Const:
                 return "const";
             case QShaderLanguage::Input:
-                return "varying";
+                if (format.shaderType() == QShaderFormat::Vertex)
+                    return "attribute";
+                else
+                    return "varying";
             case QShaderLanguage::Output:
                 return ""; // Although fragment shaders for <=2 only have fixed outputs
             case QShaderLanguage::Uniform:
@@ -314,12 +320,22 @@ QByteArray QShaderGenerator::createShaderCode(const QStringList &enabledLayers)
                            [enabledLayers] (const QString &s) { return enabledLayers.contains(s); });
     };
 
+    QVector<QString> globalInputVariables;
+    const QRegularExpression globalInputExtractRegExp(QStringLiteral("^.*\\s+(\\w+).*;$"));
+
     const QVector<QShaderNode> nodes = graph.nodes();
     for (const QShaderNode &node : nodes) {
         if (intersectsEnabledLayers(node.layers())) {
             const QByteArrayList headerSnippets = node.rule(format).headerSnippets;
             for (const QByteArray &snippet : headerSnippets) {
                 code << replaceParameters(snippet, node, format);
+
+                // If node is an input, record the variable name into the globalInputVariables vector
+                if (node.type() == QShaderNode::Input) {
+                    const QRegularExpressionMatch match = globalInputExtractRegExp.match(QString::fromUtf8(code.last()));
+                    if (match.hasMatch())
+                        globalInputVariables.push_back(match.captured(1));
+                }
             }
         }
     }
@@ -328,16 +344,129 @@ QByteArray QShaderGenerator::createShaderCode(const QStringList &enabledLayers)
     code << QByteArrayLiteral("void main()");
     code << QByteArrayLiteral("{");
 
+    const QRegularExpression localToGlobalRegExp(QStringLiteral("^.*\\s+(\\w+)\\s*=\\s*((?:\\w+\\(.*\\))|(?:\\w+)).*;$"));
+    const QRegularExpression temporaryVariableToAssignmentRegExp(QStringLiteral("^(.*\\s+(v\\d+))\\s*=\\s*(.*);$"));
+    const QRegularExpression temporaryVariableInAssignmentRegExp(QStringLiteral("\\W*(v\\d+)\\W*"));
+    const QRegularExpression outputToTemporaryAssignmentRegExp(QStringLiteral("^\\s*(\\w+)\\s*=\\s*(.*);$"));
+
+    struct Variable;
+
+    struct Assignment
+    {
+        QString expression;
+        QVector<Variable *> referencedVariables;
+    };
+
+    struct Variable
+    {
+        enum Type {
+            GlobalInput,
+            TemporaryAssignment,
+            Output
+        };
+
+        QString name;
+        QString declaration;
+        int referenceCount = 0;
+        Assignment assignment;
+        Type type = TemporaryAssignment;
+        bool substituted = false;
+
+        static void substitute(Variable *v)
+        {
+            if (v->substituted)
+                return;
+
+            qCDebug(ShaderGenerator) << "Begin Substituting " << v->name << " = " << v->assignment.expression;
+            for (Variable *ref : qAsConst(v->assignment.referencedVariables)) {
+                // Recursively substitute
+                Variable::substitute(ref);
+
+                // Replace all variables referenced only once in the assignment
+                // by their actual expression
+                if (ref->referenceCount == 1 || ref->type == Variable::GlobalInput) {
+                    const QRegularExpression r(QStringLiteral("(.*\\b)(%1)(\\b.*)").arg(ref->name));
+                    if (v->assignment.referencedVariables.size() == 1)
+                        v->assignment.expression.replace(r,
+                                                         QStringLiteral("\\1%2\\3").arg(ref->assignment.expression));
+                    else
+                        v->assignment.expression.replace(r,
+                                                         QStringLiteral("(\\1%2\\3)").arg(ref->assignment.expression));
+                }
+            }
+            qCDebug(ShaderGenerator) << "Done Substituting " << v->name << " = " << v->assignment.expression;
+            v->substituted = true;
+        }
+    };
+
+    struct LineContent
+    {
+        QByteArray rawContent;
+        Variable *var = nullptr;
+    };
+
+    // Table to store temporary variables that should be replaced:
+    // - If variable references a a global variables
+    //   -> we will use the global variable directly
+    // - If variable references a function results
+    //   -> will be kept only if variable is referenced more than once.
+    // This avoids having vec3 v56 = vertexPosition; when we could
+    // just use vertexPosition directly.
+    // The added benefit is when having arrays, we don't try to create
+    // mat4 v38 = skinningPalelette[100] which would be invalid
+    QVector<Variable> temporaryVariables;
+    // Reserve more than enough space to ensure no reallocation will take place
+    temporaryVariables.reserve(nodes.size() * 8);
+
+    QVector<LineContent> lines;
+
+    auto createVariable = [&] () -> Variable * {
+        Q_ASSERT(temporaryVariables.capacity() > 0);
+        temporaryVariables.resize(temporaryVariables.size() + 1);
+        return &temporaryVariables.last();
+    };
+
+    auto findVariable = [&] (const QString &name) -> Variable * {
+        const auto end = temporaryVariables.end();
+        auto it = std::find_if(temporaryVariables.begin(), end,
+                               [=] (const Variable &a) { return a.name == name; });
+        if (it != end)
+            return &(*it);
+        return nullptr;
+    };
+
+    auto gatherTemporaryVariablesFromAssignment = [&] (Variable *v, const QString &assignmentContent) {
+        QRegularExpressionMatchIterator subMatchIt = temporaryVariableInAssignmentRegExp.globalMatch(assignmentContent);
+        while (subMatchIt.hasNext()) {
+            const QRegularExpressionMatch subMatch = subMatchIt.next();
+            const QString variableName = subMatch.captured(1);
+
+            // Variable we care about should already exists -> an expression cannot reference a variable that hasn't been defined
+            Variable *u = findVariable(variableName);
+            Q_ASSERT(u);
+
+            // Increase reference count for u
+            ++u->referenceCount;
+            // Insert u as reference for variable v
+            v->assignment.referencedVariables.push_back(u);
+        }
+    };
+
     for (const QShaderGraph::Statement &statement : graph.createStatements(enabledLayers)) {
         const QShaderNode node = statement.node;
         QByteArray line = node.rule(format).substitution;
         const QVector<QShaderNodePort> ports = node.ports();
+
+        // Generate temporary variable names vN
         for (const QShaderNodePort &port : ports) {
             const QString portName = port.name;
             const QShaderNodePort::Direction portDirection = port.direction;
             const bool isInput = port.direction == QShaderNodePort::Input;
 
             const int portIndex = statement.portIndex(portDirection, portName);
+
+            Q_ASSERT(portIndex >= 0);
+
             const int variableIndex = isInput ? statement.inputs.at(portIndex)
                                               : statement.outputs.at(portIndex);
             if (variableIndex < 0)
@@ -345,14 +474,120 @@ QByteArray QShaderGenerator::createShaderCode(const QStringList &enabledLayers)
 
             const auto placeholder = QByteArray(QByteArrayLiteral("$") + portName.toUtf8());
             const auto variable = QByteArray(QByteArrayLiteral("v") + QByteArray::number(variableIndex));
+
             line.replace(placeholder, variable);
         }
 
-        code << QByteArrayLiteral("    ") + replaceParameters(line, node, format);
+        // Substitute variable names by generated vN variable names
+        const QByteArray substitutionedLine = replaceParameters(line, node, format);
+
+        Variable *v = nullptr;
+
+        switch (node.type()) {
+        // Record name of temporary variable that possibly references a global input
+        // We will replace the temporary variables by the matching global variables later
+        case QShaderNode::Input: {
+            const QRegularExpressionMatch match = localToGlobalRegExp.match(QString::fromUtf8(substitutionedLine));
+            if (match.hasMatch()) {
+                const QString localVariable = match.captured(1);
+                const QString globalVariable = match.captured(2);
+
+                v = createVariable();
+                v->name = localVariable;
+                v->type = Variable::GlobalInput;
+
+                Assignment assignment;
+                assignment.expression = globalVariable;
+                v->assignment = assignment;
+            }
+            break;
+        }
+
+        case QShaderNode::Function: {
+            const QRegularExpressionMatch match = temporaryVariableToAssignmentRegExp.match(QString::fromUtf8(substitutionedLine));
+            if (match.hasMatch()) {
+                const QString localVariableDeclaration = match.captured(1);
+                const QString localVariableName = match.captured(2);
+                const QString assignmentContent = match.captured(3);
+
+                // Add new variable -> it cannot exist already
+                v = createVariable();
+                v->name = localVariableName;
+                v->declaration = localVariableDeclaration;
+                v->assignment.expression = assignmentContent;
+
+                // Find variables that may be referenced in the assignment
+                gatherTemporaryVariablesFromAssignment(v, assignmentContent);
+            }
+            break;
+        }
+
+        case QShaderNode::Output: {
+            const QRegularExpressionMatch match = outputToTemporaryAssignmentRegExp.match(QString::fromUtf8(substitutionedLine));
+            if (match.hasMatch()) {
+                const QString outputDeclaration = match.captured(1);
+                const QString assignmentContent = match.captured(2);
+
+                v = createVariable();
+                v->name = outputDeclaration;
+                v->declaration = outputDeclaration;
+                v->type = Variable::Output;
+
+                Assignment assignment;
+                assignment.expression = assignmentContent;
+                v->assignment = assignment;
+
+                // Find variables that may be referenced in the assignment
+                gatherTemporaryVariablesFromAssignment(v, assignmentContent);
+            }
+            break;
+        }
+        case QShaderNode::Invalid:
+            break;
+        }
+
+        LineContent lineContent;
+        lineContent.rawContent = QByteArray(QByteArrayLiteral("    ") + substitutionedLine);
+        lineContent.var = v;
+        lines << lineContent;
+    }
+
+    // Go through all lines
+    // Perform substitution of line with temporary variables substitution
+    for (LineContent &lineContent : lines) {
+        Variable *v = lineContent.var;
+        qCDebug(ShaderGenerator) << lineContent.rawContent;
+        if (v != nullptr) {
+            Variable::substitute(v);
+
+            qCDebug(ShaderGenerator) << "Line " << lineContent.rawContent << "is assigned to temporary" << v->name;
+
+            // Check number of occurrences a temporary variable is referenced
+            if (v->referenceCount == 1 || v->type == Variable::GlobalInput) {
+                // If it is referenced only once, no point in creating a temporary
+                // Clear content for current line
+                lineContent.rawContent.clear();
+                // We assume expression that were referencing vN will have vN properly substituted
+            } else {
+                lineContent.rawContent = QStringLiteral("    %1 = %2;").arg(v->declaration)
+                                                                       .arg(v->assignment.expression)
+                                                                       .toUtf8();
+            }
+
+            qCDebug(ShaderGenerator) << "Updated Line is " << lineContent.rawContent;
+        }
+    }
+
+    // Go throug all lines and insert content
+    for (const LineContent &lineContent : qAsConst(lines)) {
+        if (!lineContent.rawContent.isEmpty()) {
+            code << lineContent.rawContent;
+        }
     }
 
     code << QByteArrayLiteral("}");
     code << QByteArray();
+
     return code.join('\n');
 }
 
diff --git a/src/gui/util/qshadergenerator_p.h b/src/gui/util/qshadergenerator_p.h
index 7bc8838b52..1f6f9d2532 100644
--- a/src/gui/util/qshadergenerator_p.h
+++ b/src/gui/util/qshadergenerator_p.h
@@ -54,9 +54,13 @@
 #include <QtGui/private/qtguiglobal_p.h>
 
 #include <QtGui/private/qshadergraph_p.h>
+#include <QtCore/QLoggingCategory>
+
 
 QT_BEGIN_NAMESPACE
 
+Q_DECLARE_LOGGING_CATEGORY(ShaderGenerator)
+
 class QShaderGenerator
 {
 public:
diff --git a/src/gui/util/qshadernodesloader.cpp b/src/gui/util/qshadernodesloader.cpp
index af565c906d..922479332c 100644
--- a/src/gui/util/qshadernodesloader.cpp
+++ b/src/gui/util/qshadernodesloader.cpp
@@ -251,6 +251,17 @@ void QShaderNodesLoader::load(const QJsonObject &prototypesObject)
                     break;
                 }
 
+                // We default out to a Fragment ShaderType if nothing is specified
+                // as that was the initial behavior we introduced
+                const QString shaderType = formatObject.value(QStringLiteral("shaderType")).toString();
+                format.setShaderType(shaderType == QStringLiteral("Fragment") ? QShaderFormat::Fragment
+                                   : shaderType == QStringLiteral("Vertex") ? QShaderFormat::Vertex
+                                   : shaderType == QStringLiteral("TessellationControl") ? QShaderFormat::TessellationControl
+                                   : shaderType == QStringLiteral("TessellationEvaluation") ? QShaderFormat::TessellationEvaluation
+                                   : shaderType == QStringLiteral("Geometry") ? QShaderFormat::Geometry
+                                   : shaderType == QStringLiteral("Compute") ? QShaderFormat::Compute
+                                   : QShaderFormat::Fragment);
+
                 const QByteArray substitution = substitutionValue.toString().toUtf8();
 
                 const QJsonValue snippetsValue = ruleObject.value(QStringLiteral("headerSnippets"));
diff --git a/src/gui/util/util.pri b/src/gui/util/util.pri
index e5e711b1a0..d3402133d6 100644
--- a/src/gui/util/util.pri
+++ b/src/gui/util/util.pri
@@ -8,7 +8,6 @@ HEADERS += \
         util/qabstractlayoutstyleinfo_p.h \
         util/qlayoutpolicy_p.h \
         util/qshaderformat_p.h \
-        util/qshadergenerator_p.h \
         util/qshadergraph_p.h \
         util/qshadergraphloader_p.h \
         util/qshaderlanguage_p.h \
@@ -29,7 +28,6 @@ SOURCES += \
         util/qabstractlayoutstyleinfo.cpp \
         util/qlayoutpolicy.cpp \
         util/qshaderformat.cpp \
-        util/qshadergenerator.cpp \
         util/qshadergraph.cpp \
         util/qshadergraphloader.cpp \
         util/qshaderlanguage.cpp \
@@ -41,3 +39,10 @@ SOURCES += \
         util/qpkmhandler.cpp \
         util/qktxhandler.cpp \
         util/qastchandler.cpp
+
+qtConfig(regularexpression) {
+    HEADERS += \
+        util/qshadergenerator_p.h
+    SOURCES += \
+        util/qshadergenerator.cpp
+}