From 389dec3e7ca530d7d6944772a1152d130cfb8e70 Mon Sep 17 00:00:00 2001 From: Paul Lemire Date: Thu, 11 Apr 2019 10:28:49 +0200 Subject: Only generate temporaries when it makes sense - Never for global inputs - Otherwise only if the temporary is referenced more than once -> meaning it's actually caching the result of some operation Tests updated accordingly. Change-Id: Ic76615370d23dee3965ca6350d5257a8be5a3e22 Reviewed-by: Sean Harmer --- src/gui/util/qshadergenerator.cpp | 234 ++++++++++++++++++--- src/gui/util/qshadergenerator_p.h | 4 + .../util/qshadergenerator/tst_qshadergenerator.cpp | 214 ++++++++++++++++--- 3 files changed, 391 insertions(+), 61 deletions(-) diff --git a/src/gui/util/qshadergenerator.cpp b/src/gui/util/qshadergenerator.cpp index 205118f41c..bcb985de54 100644 --- a/src/gui/util/qshadergenerator.cpp +++ b/src/gui/util/qshadergenerator.cpp @@ -44,6 +44,8 @@ QT_BEGIN_NAMESPACE +Q_LOGGING_CATEGORY(ShaderGenerator, "ShaderGenerator", QtWarningMsg) + namespace { QByteArray toGlsl(QShaderLanguage::StorageQualifier qualifier, const QShaderFormat &format) @@ -342,18 +344,120 @@ QByteArray QShaderGenerator::createShaderCode(const QStringList &enabledLayers) code << QByteArrayLiteral("void main()"); code << QByteArrayLiteral("{"); - // Table to store temporary variables that should be replaced by global - // variables. This avoids having vec3 v56 = vertexPosition; when we could + const QRegularExpression localToGlobalRegExp(QStringLiteral("^.*\\s+(\\w+)\\s*=\\s*((?:\\w+\\(.*\\))|(?:\\w+)).*;$")); + const QRegularExpression temporaryVariableToAssignmentRegExp(QStringLiteral("^(.*\\s+(v\\d+))\\s*=\\s*(.*);$")); + const QRegularExpression temporaryVariableInAssignmentRegExp(QStringLiteral("\\W*(v\\d+)\\W*")); + const QRegularExpression outputToTemporaryAssignmentRegExp(QStringLiteral("^\\s*(\\w+)\\s*=\\s*(.*);$")); + + struct Variable; + + struct Assignment + { + QString expression; + QVector referencedVariables; + }; + + struct Variable + { + enum Type { + GlobalInput, + TemporaryAssignment, + Output + }; + + QString name; + QString declaration; + int referenceCount = 0; + Assignment assignment; + Type type = TemporaryAssignment; + bool substituted = false; + + static void substitute(Variable *v) + { + if (v->substituted) + return; + + qCDebug(ShaderGenerator) << "Begin Substituting " << v->name << " = " << v->assignment.expression; + for (Variable *ref : qAsConst(v->assignment.referencedVariables)) { + // Recursively substitute + Variable::substitute(ref); + + // Replace all variables referenced only once in the assignment + // by their actual expression + if (ref->referenceCount == 1 || ref->type == Variable::GlobalInput) { + const QRegularExpression r(QStringLiteral("(.*\\b)(%1)(\\b.*)").arg(ref->name)); + if (v->assignment.referencedVariables.size() == 1) + v->assignment.expression.replace(r, + QStringLiteral("\\1%2\\3").arg(ref->assignment.expression)); + else + v->assignment.expression.replace(r, + QStringLiteral("(\\1%2\\3)").arg(ref->assignment.expression)); + } + } + qCDebug(ShaderGenerator) << "Done Substituting " << v->name << " = " << v->assignment.expression; + v->substituted = true; + } + }; + + struct LineContent + { + QByteArray rawContent; + Variable *var = nullptr; + }; + + // Table to store temporary variables that should be replaced: + // - If variable references a a global variables + // -> we will use the global variable directly + // - If variable references a function results + // -> will be kept only if variable is referenced more than once. + // This avoids having vec3 v56 = vertexPosition; when we could // just use vertexPosition directly. // The added benefit is when having arrays, we don't try to create // mat4 v38 = skinningPalelette[100] which would be invalid - QHash localReferencesToGlobalInputs; - const QRegularExpression localToGlobalRegExp(QStringLiteral("^.*\\s+(\\w+)\\s*=\\s*(\\w+).*;$")); + QVector temporaryVariables; + // Reserve more than enough space to ensure no reallocation will take place + temporaryVariables.reserve(nodes.size() * 8); + + QVector lines; + + auto createVariable = [&] () -> Variable * { + Q_ASSERT(temporaryVariables.capacity() > 0); + temporaryVariables.resize(temporaryVariables.size() + 1); + return &temporaryVariables.last(); + }; + + auto findVariable = [&] (const QString &name) -> Variable * { + const auto end = temporaryVariables.end(); + auto it = std::find_if(temporaryVariables.begin(), end, + [=] (const Variable &a) { return a.name == name; }); + if (it != end) + return &(*it); + return nullptr; + }; + + auto gatherTemporaryVariablesFromAssignment = [&] (Variable *v, const QString &assignmentContent) { + QRegularExpressionMatchIterator subMatchIt = temporaryVariableInAssignmentRegExp.globalMatch(assignmentContent); + while (subMatchIt.hasNext()) { + const QRegularExpressionMatch subMatch = subMatchIt.next(); + const QString variableName = subMatch.captured(1); + + // Variable we care about should already exists -> an expression cannot reference a variable that hasn't been defined + Variable *u = findVariable(variableName); + Q_ASSERT(u); + + // Increase reference count for u + ++u->referenceCount; + // Insert u as reference for variable v + v->assignment.referencedVariables.push_back(u); + } + }; for (const QShaderGraph::Statement &statement : graph.createStatements(enabledLayers)) { const QShaderNode node = statement.node; QByteArray line = node.rule(format).substitution; const QVector ports = node.ports(); + + // Generate temporary variable names vN for (const QShaderNodePort &port : ports) { const QString portName = port.name; const QShaderNodePort::Direction portDirection = port.direction; @@ -374,47 +478,117 @@ QByteArray QShaderGenerator::createShaderCode(const QStringList &enabledLayers) line.replace(placeholder, variable); } + // Substitute variable names by generated vN variable names const QByteArray substitutionedLine = replaceParameters(line, node, format); + Variable *v = nullptr; + + switch (node.type()) { // Record name of temporary variable that possibly references a global input // We will replace the temporary variables by the matching global variables later - bool isAGlobalInputVariable = false; - if (node.type() == QShaderNode::Input) { + case QShaderNode::Input: { const QRegularExpressionMatch match = localToGlobalRegExp.match(QString::fromUtf8(substitutionedLine)); if (match.hasMatch()) { + const QString localVariable = match.captured(1); const QString globalVariable = match.captured(2); - if (globalInputVariables.contains(globalVariable)) { - const QString localVariable = match.captured(1); - // TO DO: Clean globalVariable (remove brackets ...) - localReferencesToGlobalInputs.insert(localVariable, globalVariable); - isAGlobalInputVariable = true; - } + + v = createVariable(); + v->name = localVariable; + v->type = Variable::GlobalInput; + + Assignment assignment; + assignment.expression = globalVariable; + v->assignment = assignment; + } + break; + } + + case QShaderNode::Function: { + const QRegularExpressionMatch match = temporaryVariableToAssignmentRegExp.match(QString::fromUtf8(substitutionedLine)); + if (match.hasMatch()) { + const QString localVariableDeclaration = match.captured(1); + const QString localVariableName = match.captured(2); + const QString assignmentContent = match.captured(3); + + // Add new variable -> it cannot exist already + v = createVariable(); + v->name = localVariableName; + v->declaration = localVariableDeclaration; + v->assignment.expression = assignmentContent; + + // Find variables that may be referenced in the assignment + gatherTemporaryVariablesFromAssignment(v, assignmentContent); + } + break; + } + + case QShaderNode::Output: { + const QRegularExpressionMatch match = outputToTemporaryAssignmentRegExp.match(QString::fromUtf8(substitutionedLine)); + if (match.hasMatch()) { + const QString outputDeclaration = match.captured(1); + const QString assignmentContent = match.captured(2); + + v = createVariable(); + v->name = outputDeclaration; + v->declaration = outputDeclaration; + v->type = Variable::Output; + + Assignment assignment; + assignment.expression = assignmentContent; + v->assignment = assignment; + + // Find variables that may be referenced in the assignment + gatherTemporaryVariablesFromAssignment(v, assignmentContent); } + break; + } + case QShaderNode::Invalid: + break; } - // Only insert content for lines aren't inputs or have not matching - // globalVariables for now - if (!isAGlobalInputVariable) - code << QByteArrayLiteral(" ") + substitutionedLine; + LineContent lineContent; + lineContent.rawContent = QByteArray(QByteArrayLiteral(" ") + substitutionedLine); + lineContent.var = v; + lines << lineContent; } - code << QByteArrayLiteral("}"); - code << QByteArray(); + // Go through all lines + // Perform substitution of line with temporary variables substitution + for (LineContent &lineContent : lines) { + Variable *v = lineContent.var; + qCDebug(ShaderGenerator) << lineContent.rawContent; + if (v != nullptr) { + Variable::substitute(v); + + qCDebug(ShaderGenerator) << "Line " << lineContent.rawContent << "is assigned to temporary" << v->name; + + // Check number of occurrences a temporary variable is referenced + if (v->referenceCount == 1 || v->type == Variable::GlobalInput) { + // If it is referenced only once, no point in creating a temporary + // Clear content for current line + lineContent.rawContent.clear(); + // We assume expression that were referencing vN will have vN properly substituted + } else { + lineContent.rawContent = QStringLiteral(" %1 = %2;").arg(v->declaration) + .arg(v->assignment.expression) + .toUtf8(); + } - // Replace occurrences of local variables which reference a global variable - // by the global variables directly - auto it = localReferencesToGlobalInputs.cbegin(); - const auto end = localReferencesToGlobalInputs.cend(); - QString codeString = QString::fromUtf8(code.join('\n')); - - while (it != end) { - const QRegularExpression r(QStringLiteral("\\b(%1)([\\b|\\.|;|\\)|\\[|\\s|\\*|\\+|\\/|\\-|,])").arg(it.key()), - QRegularExpression::MultilineOption); - codeString.replace(r, QStringLiteral("%1\\2").arg(it.value())); - ++it; + qCDebug(ShaderGenerator) << "Updated Line is " << lineContent.rawContent; + } } - return codeString.toUtf8(); + // Go throug all lines and insert content + for (const LineContent &lineContent : qAsConst(lines)) { + if (!lineContent.rawContent.isEmpty()) { + code << lineContent.rawContent; + } + } + + code << QByteArrayLiteral("}"); + code << QByteArray(); + + return code.join('\n'); } QT_END_NAMESPACE diff --git a/src/gui/util/qshadergenerator_p.h b/src/gui/util/qshadergenerator_p.h index 7bc8838b52..1f6f9d2532 100644 --- a/src/gui/util/qshadergenerator_p.h +++ b/src/gui/util/qshadergenerator_p.h @@ -54,9 +54,13 @@ #include #include +#include + QT_BEGIN_NAMESPACE +Q_DECLARE_LOGGING_CATEGORY(ShaderGenerator) + class QShaderGenerator { public: diff --git a/tests/auto/gui/util/qshadergenerator/tst_qshadergenerator.cpp b/tests/auto/gui/util/qshadergenerator/tst_qshadergenerator.cpp index f8bb0c3851..59e93d127f 100644 --- a/tests/auto/gui/util/qshadergenerator/tst_qshadergenerator.cpp +++ b/tests/auto/gui/util/qshadergenerator/tst_qshadergenerator.cpp @@ -137,7 +137,7 @@ namespace createPort(QShaderNodePort::Output, "color") }); sampleTexture.addRule(openGLES2, QShaderNode::Rule("highp vec4 $color = texture2D($sampler, $coord);")); - sampleTexture.addRule(openGL3, QShaderNode::Rule("vec4 $color = texture2D($sampler, $coord);")); + sampleTexture.addRule(openGL3, QShaderNode::Rule("vec4 $color = texture($sampler, $coord);")); auto lightFunction = createNode({ createPort(QShaderNodePort::Input, "baseColor"), @@ -197,6 +197,7 @@ private slots: void shouldProcessLanguageQualifierAndTypeEnums(); void shouldGenerateDifferentCodeDependingOnActiveLayers(); void shouldUseGlobalVariableRatherThanTemporaries(); + void shouldGenerateTemporariesWisely(); }; void tst_QShaderGenerator::shouldHaveDefaultState() @@ -237,10 +238,7 @@ void tst_QShaderGenerator::shouldGenerateShaderCode_data() << "" << "void main()" << "{" - << " highp vec4 v5 = texture2D(texture, texCoord);" - << " highp vec4 v6 = lightModel(v5, worldPosition, lightIntensity);" - << " highp vec4 v7 = v6 * pow(2.0, exposure);" - << " gl_fragColor = v7;" + << " gl_fragColor = (((((lightModel(((texture2D(texture, texCoord))), worldPosition, lightIntensity)))) * pow(2.0, exposure)));" << "}" << ""; @@ -254,10 +252,7 @@ void tst_QShaderGenerator::shouldGenerateShaderCode_data() << "" << "void main()" << "{" - << " vec4 v5 = texture2D(texture, texCoord);" - << " vec4 v6 = lightModel(v5, worldPosition, lightIntensity);" - << " vec4 v7 = v6 * pow(2.0, exposure);" - << " fragColor = v7;" + << " fragColor = (((((lightModel(((texture(texture, texCoord))), worldPosition, lightIntensity)))) * pow(2.0, exposure)));" << "}" << ""; @@ -869,8 +864,7 @@ void tst_QShaderGenerator::shouldGenerateDifferentCodeDependingOnActiveLayers() << "" << "void main()" << "{" - << " vec4 v2 = lightModel(diffuseUniform, normalUniform);" - << " fragColor = v2;" + << " fragColor = ((lightModel(diffuseUniform, normalUniform)));" << "}" << ""; QCOMPARE(code, expected.join("\n")); @@ -892,9 +886,7 @@ void tst_QShaderGenerator::shouldGenerateDifferentCodeDependingOnActiveLayers() << "" << "void main()" << "{" - << " vec3 v2 = texture2D(normalTexture, texCoord).rgb;" - << " vec4 v3 = lightModel(diffuseUniform, v2);" - << " fragColor = v3;" + << " fragColor = ((lightModel(diffuseUniform, texture2D(normalTexture, texCoord).rgb)));" << "}" << ""; QCOMPARE(code, expected.join("\n")); @@ -916,9 +908,7 @@ void tst_QShaderGenerator::shouldGenerateDifferentCodeDependingOnActiveLayers() << "" << "void main()" << "{" - << " vec4 v1 = texture2D(diffuseTexture, texCoord);" - << " vec4 v3 = lightModel(v1, normalUniform);" - << " fragColor = v3;" + << " fragColor = ((lightModel(texture2D(diffuseTexture, texCoord), normalUniform)));" << "}" << ""; QCOMPARE(code, expected.join("\n")); @@ -940,10 +930,7 @@ void tst_QShaderGenerator::shouldGenerateDifferentCodeDependingOnActiveLayers() << "" << "void main()" << "{" - << " vec3 v2 = texture2D(normalTexture, texCoord).rgb;" - << " vec4 v1 = texture2D(diffuseTexture, texCoord);" - << " vec4 v3 = lightModel(v1, v2);" - << " fragColor = v3;" + << " fragColor = ((lightModel(texture2D(diffuseTexture, texCoord), texture2D(normalTexture, texCoord).rgb)));" << "}" << ""; QCOMPARE(code, expected.join("\n")); @@ -967,13 +954,13 @@ void tst_QShaderGenerator::shouldUseGlobalVariableRatherThanTemporaries() createPort(QShaderNodePort::Input, "varName"), createPort(QShaderNodePort::Output, "out") }); - fakeMultiPlyNoSpace.addRule(gl4, QShaderNode::Rule("vec4 $out = $varName*v11;")); + fakeMultiPlyNoSpace.addRule(gl4, QShaderNode::Rule("vec4 $out = $varName*speed;")); auto fakeMultiPlySpace = createNode({ createPort(QShaderNodePort::Input, "varName"), createPort(QShaderNodePort::Output, "out") }); - fakeMultiPlySpace.addRule(gl4, QShaderNode::Rule("vec4 $out = $varName * v11;")); + fakeMultiPlySpace.addRule(gl4, QShaderNode::Rule("vec4 $out = $varName * speed;")); auto fakeJoinNoSpace = createNode({ createPort(QShaderNodePort::Input, "varName"), @@ -1063,20 +1050,185 @@ void tst_QShaderGenerator::shouldUseGlobalVariableRatherThanTemporaries() << "" << "void main()" << "{" - << " vec4 v7 = vertexPosition / vertexPosition;" - << " vec4 v6 = vertexPosition.xyzw - vertexPosition;" - << " vec4 v5 = vertexPosition.xyzw + vertexPosition;" - << " vec4 v4 = vec4(vertexPosition.xyz, vertexPosition.w);" - << " vec4 v3 = vec4(vertexPosition.xyz,vertexPosition.w);" - << " vec4 v2 = vertexPosition * v11;" - << " vec4 v1 = vertexPosition*v11;" - << " fragColor = v1 + v2 + v3 + v4 + v5 + v6 + v7;" + << " fragColor = (((((((vertexPosition*speed + vertexPosition * speed + ((vec4(vertexPosition.xyz,vertexPosition.w))) + ((vec4(vertexPosition.xyz, vertexPosition.w))) + ((vertexPosition.xyzw + vertexPosition)) + ((vertexPosition.xyzw - vertexPosition)) + ((vertexPosition / vertexPosition)))))))));" << "}" << ""; QCOMPARE(code, expected.join("\n")); } } +void tst_QShaderGenerator::shouldGenerateTemporariesWisely() +{ + // GIVEN + const auto gl4 = createFormat(QShaderFormat::OpenGLCoreProfile, 4, 0); + + { + auto attribute = createNode({ + createPort(QShaderNodePort::Output, "vertexPosition") + }); + attribute.addRule(gl4, QShaderNode::Rule("vec4 $vertexPosition = vertexPosition;", + QByteArrayList() << "in vec4 vertexPosition;")); + + auto complexFunction = createNode({ + createPort(QShaderNodePort::Input, "inputVarName"), + createPort(QShaderNodePort::Output, "out") + }); + complexFunction.addRule(gl4, QShaderNode::Rule("vec4 $out = $inputVarName * 2.0;")); + + auto complexFunction2 = createNode({ + createPort(QShaderNodePort::Input, "inputVarName"), + createPort(QShaderNodePort::Output, "out") + }); + complexFunction2.addRule(gl4, QShaderNode::Rule("vec4 $out = $inputVarName * 4.0;")); + + auto complexFunction3 = createNode({ + createPort(QShaderNodePort::Input, "a"), + createPort(QShaderNodePort::Input, "b"), + createPort(QShaderNodePort::Output, "out") + }); + complexFunction3.addRule(gl4, QShaderNode::Rule("vec4 $out = $a + $b;")); + + auto shaderOutput1 = createNode({ + createPort(QShaderNodePort::Input, "input") + }); + + shaderOutput1.addRule(gl4, QShaderNode::Rule("shaderOutput1 = $input;", + QByteArrayList() << "out vec4 shaderOutput1;")); + + auto shaderOutput2 = createNode({ + createPort(QShaderNodePort::Input, "input") + }); + + shaderOutput2.addRule(gl4, QShaderNode::Rule("shaderOutput2 = $input;", + QByteArrayList() << "out vec4 shaderOutput2;")); + + { + // WHEN + const auto graph = [=] { + auto res = QShaderGraph(); + + res.addNode(attribute); + res.addNode(complexFunction); + res.addNode(shaderOutput1); + + res.addEdge(createEdge(attribute.uuid(), "vertexPosition", complexFunction.uuid(), "inputVarName")); + res.addEdge(createEdge(complexFunction.uuid(), "out", shaderOutput1.uuid(), "input")); + + return res; + }(); + + auto generator = QShaderGenerator(); + generator.graph = graph; + generator.format = gl4; + + const auto code = generator.createShaderCode(); + + // THEN + const auto expected = QByteArrayList() + << "#version 400 core" + << "" + << "in vec4 vertexPosition;" + << "out vec4 shaderOutput1;" + << "" + << "void main()" + << "{" + << " shaderOutput1 = vertexPosition * 2.0;" + << "}" + << ""; + QCOMPARE(code, expected.join("\n")); + } + + { + // WHEN + const auto graph = [=] { + auto res = QShaderGraph(); + + res.addNode(attribute); + res.addNode(complexFunction); + res.addNode(shaderOutput1); + res.addNode(shaderOutput2); + + res.addEdge(createEdge(attribute.uuid(), "vertexPosition", complexFunction.uuid(), "inputVarName")); + res.addEdge(createEdge(complexFunction.uuid(), "out", shaderOutput1.uuid(), "input")); + res.addEdge(createEdge(complexFunction.uuid(), "out", shaderOutput2.uuid(), "input")); + + return res; + }(); + + auto generator = QShaderGenerator(); + generator.graph = graph; + generator.format = gl4; + + const auto code = generator.createShaderCode(); + + // THEN + const auto expected = QByteArrayList() + << "#version 400 core" + << "" + << "in vec4 vertexPosition;" + << "out vec4 shaderOutput1;" + << "out vec4 shaderOutput2;" + << "" + << "void main()" + << "{" + << " vec4 v1 = vertexPosition * 2.0;" + << " shaderOutput2 = v1;" + << " shaderOutput1 = v1;" + << "}" + << ""; + QCOMPARE(code, expected.join("\n")); + } + + { + // WHEN + const auto graph = [=] { + auto res = QShaderGraph(); + + res.addNode(attribute); + res.addNode(complexFunction); + res.addNode(complexFunction2); + res.addNode(complexFunction3); + res.addNode(shaderOutput1); + res.addNode(shaderOutput2); + + res.addEdge(createEdge(attribute.uuid(), "vertexPosition", complexFunction.uuid(), "inputVarName")); + res.addEdge(createEdge(attribute.uuid(), "vertexPosition", complexFunction2.uuid(), "inputVarName")); + + res.addEdge(createEdge(complexFunction.uuid(), "out", complexFunction3.uuid(), "a")); + res.addEdge(createEdge(complexFunction2.uuid(), "out", complexFunction3.uuid(), "b")); + + res.addEdge(createEdge(complexFunction3.uuid(), "out", shaderOutput1.uuid(), "input")); + res.addEdge(createEdge(complexFunction2.uuid(), "out", shaderOutput2.uuid(), "input")); + + return res; + }(); + + auto generator = QShaderGenerator(); + generator.graph = graph; + generator.format = gl4; + + const auto code = generator.createShaderCode(); + + // THEN + const auto expected = QByteArrayList() + << "#version 400 core" + << "" + << "in vec4 vertexPosition;" + << "out vec4 shaderOutput1;" + << "out vec4 shaderOutput2;" + << "" + << "void main()" + << "{" + << " vec4 v2 = vertexPosition * 4.0;" + << " shaderOutput2 = v2;" + << " shaderOutput1 = (vertexPosition * 2.0 + v2);" + << "}" + << ""; + QCOMPARE(code, expected.join("\n")); + } + } +} + QTEST_MAIN(tst_QShaderGenerator) #include "tst_qshadergenerator.moc" -- cgit v1.2.3