effects: Handle depth/stencil commands

Change-Id: I947d7d39651a92dec7a35175c9f1e0d1823206f3
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

4 files changed, 172 insertions, 4 deletions

diff --git a/src/runtime/q3dseffect.cpp b/src/runtime/q3dseffect.cpp
index b313dcb..c531e26 100644
--- a/src/runtime/q3dseffect.cpp
+++ b/src/runtime/q3dseffect.cpp
@@ -369,7 +369,9 @@ void Q3DSEffectParser::parsePass(Q3DSEffect &effect)
                     if (Q3DS::convertToInt32(attribute.value(), &stencilValue, "stencil value", r))
                         depthStencil.data()->stencilValue = stencilValue;
                 } else if (attribute.name() == QStringLiteral("flags")) {
-                    depthStencil.data()->flags = attribute.value().toString();
+                    Q3DSMaterial::DepthStencilFlags flags;
+                    if (Q3DSMaterial::convertToDepthStencilFlags(attribute.value(), &flags, "depth stencil flags", r))
+                        depthStencil.data()->flags = flags;
                 } else if (attribute.name() == QStringLiteral("mask")) {
                     qint32 mask;
                     if (Q3DS::convertToInt32(attribute.value(), &mask, "stencil mask", r))
diff --git a/src/runtime/q3dsmaterial.cpp b/src/runtime/q3dsmaterial.cpp
index d256db6..5c2d9b2 100644
--- a/src/runtime/q3dsmaterial.cpp
+++ b/src/runtime/q3dsmaterial.cpp
@@ -349,7 +349,7 @@ Shader parserShaderElement(QXmlStreamReader *r)
 {
     Q3DSMaterial::Shader shader;
     for (auto attribute : r->attributes()) {
-        if (attribute.name() == QStringLiteral("name"))
+        if (attribute.name() == QStringLiteral("name") || attribute.name() == QStringLiteral("Name"))
             shader.name = attribute.value().toString();
     }
     while (r->readNextStartElement()) {
@@ -552,6 +552,24 @@ bool convertToStencilOp(const QStringRef &value, StencilOp *type, const char *de
     return ok;
 }
 
+bool convertToDepthStencilFlags(const QStringRef &value, DepthStencilFlags *flags, const char *desc, QXmlStreamReader *reader)
+{
+    *flags = 0;
+    QStringList values = value.toString().split(QLatin1Char('|'), QString::SkipEmptyParts);
+    for (const QString &v : values) {
+        if (v == QStringLiteral("clear-stencil")) {
+            *flags |= Q3DSMaterial::ClearStencil;
+        } else if (v == QStringLiteral("clear-depth")) {
+            *flags |= Q3DSMaterial::ClearDepth;
+        } else {
+            if (reader)
+                reader->raiseError(QObject::tr("Invalid %1 \"%2\"").arg(QString::fromUtf8(desc)).arg(value.toString()));
+            return false;
+        }
+    }
+    return true;
+}
+
 bool convertToTextureFormat(const QStringRef &value, const QString &typeValue, TextureFormat *type, const char *desc, QXmlStreamReader *reader)
 {
     bool ok = false;
diff --git a/src/runtime/q3dsmaterial.h b/src/runtime/q3dsmaterial.h
index 29cb556..79ca179 100644
--- a/src/runtime/q3dsmaterial.h
+++ b/src/runtime/q3dsmaterial.h
@@ -284,6 +284,12 @@ enum StencilOp {
     Invert
 };
 
+enum DepthStencilFlag {
+    ClearDepth = 0x01,
+    ClearStencil = 0x02
+};
+Q_DECLARE_FLAGS(DepthStencilFlags, DepthStencilFlag)
+
 class Q3DSV_EXPORT PassCommand
 {
 public:
@@ -315,7 +321,7 @@ public:
         QString name;
         QString bufferName;
         quint32 stencilValue = 0;
-        QString flags;
+        DepthStencilFlags flags;
         quint32 mask = 4294967295U; // 0xffffffff
         BoolOp stencilFunction = Equal;
         StencilOp stencilFail = Keep;
@@ -362,6 +368,7 @@ bool convertToFilterType(const QStringRef &value, Q3DSMaterial::FilterType *type
 bool convertToClampType(const QStringRef &value, Q3DSMaterial::ClampType *type, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
 bool convertToBoolOp(const QStringRef &value, Q3DSMaterial::BoolOp *type, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
 bool convertToStencilOp(const QStringRef &value, Q3DSMaterial::StencilOp *type, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
+bool convertToDepthStencilFlags(const QStringRef &value, Q3DSMaterial::DepthStencilFlags *flags, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
 bool convertToTextureFormat(const QStringRef &value, const QString &typeValue, Q3DSMaterial::TextureFormat *type, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
 bool convertToBlendFunc(const QStringRef &value, Q3DSMaterial::BlendFunc *type, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
 bool convertToImageAccess(const QStringRef &value, Q3DSMaterial::ImageAccess *type, const char *desc = nullptr, QXmlStreamReader *reader = nullptr);
@@ -377,6 +384,8 @@ Q_DECLARE_TYPEINFO(Q3DSMaterial::PassCommand::Data, Q_MOVABLE_TYPE);
 Q_DECLARE_TYPEINFO(Q3DSMaterial::PassCommand, Q_MOVABLE_TYPE);
 Q_DECLARE_TYPEINFO(Q3DSMaterial::Pass, Q_MOVABLE_TYPE);
 
+Q_DECLARE_OPERATORS_FOR_FLAGS(Q3DSMaterial::DepthStencilFlags)
+
 QT_END_NAMESPACE
 
 #endif // Q3DSMATERIAL_H
diff --git a/src/runtime/q3dsscenemanager.cpp b/src/runtime/q3dsscenemanager.cpp
index 7741567..b5fe702 100644
--- a/src/runtime/q3dsscenemanager.cpp
+++ b/src/runtime/q3dsscenemanager.cpp
@@ -79,6 +79,11 @@
 #include <Qt3DRender/QShaderProgram>
 #include <Qt3DRender/QBuffer>
 #include <Qt3DRender/QBlitFramebuffer>
+#include <Qt3DRender/QStencilTest>
+#include <Qt3DRender/QStencilTestArguments>
+#include <Qt3DRender/QStencilMask>
+#include <Qt3DRender/QStencilOperation>
+#include <Qt3DRender/QStencilOperationArguments>
 
 #include <Qt3DAnimation/QClipAnimator>
 
@@ -4402,6 +4407,64 @@ static inline Qt3DRender::QParameter *makePropertyUniform(const QString &name, c
     return nullptr;
 }
 
+static inline Qt3DRender::QStencilTestArguments::StencilFunction convertToQt3DStencilFunc(Q3DSMaterial::BoolOp func)
+{
+    switch (func) {
+    case Q3DSMaterial::Never:
+        return Qt3DRender::QStencilTestArguments::Never;
+    case Q3DSMaterial::AlwaysTrue:
+        return Qt3DRender::QStencilTestArguments::Always;
+    case Q3DSMaterial::Less:
+        return Qt3DRender::QStencilTestArguments::Less;
+    case Q3DSMaterial::LessThanOrEqual:
+        return Qt3DRender::QStencilTestArguments::LessOrEqual;
+    case Q3DSMaterial::Equal:
+        return Qt3DRender::QStencilTestArguments::Equal;
+    case Q3DSMaterial::GreaterThanOrEqual:
+        return Qt3DRender::QStencilTestArguments::GreaterOrEqual;
+    case Q3DSMaterial::Greater:
+        return Qt3DRender::QStencilTestArguments::Greater;
+    case Q3DSMaterial::NotEqual:
+        return Qt3DRender::QStencilTestArguments::NotEqual;
+    default:
+        return Qt3DRender::QStencilTestArguments::Never;
+    }
+}
+
+static inline Qt3DRender::QStencilOperationArguments::Operation convertToQt3DStencilOp(Q3DSMaterial::StencilOp op)
+{
+    switch (op) {
+    case Q3DSMaterial::Keep:
+        return Qt3DRender::QStencilOperationArguments::Keep;
+    case Q3DSMaterial::Zero:
+        return Qt3DRender::QStencilOperationArguments::Zero;
+    case Q3DSMaterial::Replace:
+        return Qt3DRender::QStencilOperationArguments::Replace;
+    case Q3DSMaterial::Increment:
+        return Qt3DRender::QStencilOperationArguments::Increment;
+    case Q3DSMaterial::IncrementWrap:
+        return Qt3DRender::QStencilOperationArguments::IncrementWrap;
+    case Q3DSMaterial::Decrement:
+        return Qt3DRender::QStencilOperationArguments::Decrement;
+    case Q3DSMaterial::DecrementWrap:
+        return Qt3DRender::QStencilOperationArguments::DecrementWrap;
+    case Q3DSMaterial::Invert:
+        return Qt3DRender::QStencilOperationArguments::Invert;
+    default:
+        return Qt3DRender::QStencilOperationArguments::Keep;
+    }
+}
+
+static inline void setupStencilTest(Qt3DRender::QStencilTest *stencilTest, const Q3DSMaterial::PassCommand &cmd)
+{
+    auto d = cmd.data();
+    for (auto s : { stencilTest->front(), stencilTest->back() }) {
+        s->setComparisonMask(d->mask);
+        s->setReferenceValue(d->stencilValue);
+        s->setStencilFunction(convertToQt3DStencilFunc(d->stencilFunction));
+    }
+}
+
 void Q3DSSceneManager::finalizeEffects(Q3DSLayerNode *layer3DS)
 {
     Q3DSLayerAttached *layerData = static_cast<Q3DSLayerAttached *>(layer3DS->attached());
@@ -4497,6 +4560,11 @@ void Q3DSSceneManager::finalizeEffects(Q3DSLayerNode *layer3DS)
 
             Qt3DRender::QBlendEquation *blendFunc = nullptr;
             Qt3DRender::QBlendEquationArguments *blendArgs = nullptr;
+            Qt3DRender::QStencilTest *stencilTest = nullptr;
+            Qt3DRender::QStencilOperation *stencilOp = nullptr;
+            bool depthNeedsClear = false;
+            bool stencilNeedsClear = false;
+            QString depthStencilBufferName;
 
             for (const Q3DSMaterial::PassCommand &cmd : pass.commands) {
                 switch (cmd.type()) {
@@ -4620,6 +4688,52 @@ void Q3DSSceneManager::finalizeEffects(Q3DSLayerNode *layer3DS)
                 }
                     break;
 
+                case Q3DSMaterial::PassCommand::RenderStateType:
+                {
+                    if (cmd.data()->name == QStringLiteral("Stencil")) {
+                        bool enabled = false;
+                        if (Q3DS::convertToBool(&cmd.data()->value, &enabled)) {
+                            if (enabled) {
+                                if (!stencilTest)
+                                    stencilTest = new Qt3DRender::QStencilTest;
+                                Q3DSMaterial::PassCommand dummy; // with defaults
+                                setupStencilTest(stencilTest, dummy);
+                            } else {
+                                delete stencilTest;
+                                stencilTest = nullptr;
+                                delete stencilOp;
+                                stencilOp = nullptr;
+                            }
+                        }
+                    } else {
+                        qWarning("Effect %s: Unsupported render state %s", eff3DS->id().constData(), qPrintable(cmd.data()->name));
+                    }
+                }
+                    break;
+
+                case Q3DSMaterial::PassCommand::DepthStencilType:
+                {
+                    auto d = cmd.data();
+                    depthStencilBufferName = d->bufferName;
+                    depthNeedsClear = d->flags.testFlag(Q3DSMaterial::ClearDepth);
+                    stencilNeedsClear = d->flags.testFlag(Q3DSMaterial::ClearStencil);
+
+                    if (!stencilTest)
+                        stencilTest = new Qt3DRender::QStencilTest;
+
+                    setupStencilTest(stencilTest, cmd);
+
+                    if (!stencilOp)
+                        stencilOp = new Qt3DRender::QStencilOperation;
+
+                    for (auto s : { stencilOp->front(), stencilOp->back() }) {
+                        s->setStencilTestFailureOperation(convertToQt3DStencilOp(d->stencilFail));
+                        s->setDepthTestFailureOperation(convertToQt3DStencilOp(d->depthFail));
+                        s->setAllTestsPassOperation(convertToQt3DStencilOp(d->depthPass));
+                    }
+                }
+                    break;
+
                 default:
                     qWarning("Effect %s: Unhandled command %d", eff3DS->id().constData(), cmd.type());
                     break;
@@ -4707,20 +4821,45 @@ void Q3DSSceneManager::finalizeEffects(Q3DSLayerNode *layer3DS)
                 quadInfo.renderStates << blendFunc;
             if (blendArgs)
                 quadInfo.renderStates << blendArgs;
+            if (stencilTest)
+                quadInfo.renderStates << stencilTest;
+            if (stencilOp)
+                quadInfo.renderStates << stencilOp;
 
             buildFsQuad(quadInfo);
 
             Qt3DRender::QRenderTargetSelector *rtSel = new Qt3DRender::QRenderTargetSelector(layerData->effectData.effectRoot);
             Qt3DRender::QRenderTarget *rt = new Qt3DRender::QRenderTarget;
+            m_profiler->trackNewObject(rt, Q3DSProfiler::RenderTargetObject, "RT for effect %s pass %d",
+                                       eff3DS->id().constData(), passIdx + 1);
             Qt3DRender::QRenderTargetOutput *color = new Qt3DRender::QRenderTargetOutput;
             color->setAttachmentPoint(Qt3DRender::QRenderTargetOutput::Color0);
             color->setTexture(passOutput);
             rt->addOutput(color);
+            if (!depthStencilBufferName.isEmpty()) {
+                if (effData->textureBuffers.contains(depthStencilBufferName)) {
+                    qCDebug(lcScene, "    Binding buffer %s for depth-stencil", qPrintable(depthStencilBufferName));
+                    Qt3DRender::QRenderTargetOutput *ds = new Qt3DRender::QRenderTargetOutput;
+                    ds->setAttachmentPoint(Qt3DRender::QRenderTargetOutput::DepthStencil);
+                    ds->setTexture(effData->textureBuffers[depthStencilBufferName].texture);
+                    rt->addOutput(ds);
+                } else {
+                    qWarning("Effect %s: Unknown depth-stencil buffer %s",
+                             eff3DS->id().constData(), qPrintable(depthStencilBufferName));
+                }
+            }
             rtSel->setTarget(rt);
 
             Qt3DRender::QClearBuffers *clearBuf = new Qt3DRender::QClearBuffers(rtSel);
             clearBuf->setClearColor(Qt::transparent);
-            clearBuf->setBuffers(outputNeedsClear ? Qt3DRender::QClearBuffers::ColorBuffer : Qt3DRender::QClearBuffers::None);
+            int buffersToClear = Qt3DRender::QClearBuffers::None;
+            if (outputNeedsClear)
+                buffersToClear |= Qt3DRender::QClearBuffers::ColorBuffer;
+            if (depthNeedsClear)
+                buffersToClear |= Qt3DRender::QClearBuffers::DepthBuffer;
+            if (stencilNeedsClear)
+                buffersToClear |= Qt3DRender::QClearBuffers::StencilBuffer;
+            clearBuf->setBuffers(Qt3DRender::QClearBuffers::BufferType(buffersToClear));
 
             Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(clearBuf);
             layerFilter->addLayer(effData->quadEntityTag);