Move away from CBOR in QShaderDescription serialization

...but keep support for deserializing for all older versions in order to play
nice with existing .qsb files.

The usage of binary JSON and then CBOR is a historical artifact: relying
on the QJsonDocument (which we generate for purposes unrelated to binary
serialization) was a convenient shortcut. However, writing to and
reading from a QDataStream instead (which QShader already does) is trivial.
In order not to be limited by potential CBOR requirements in the future,
take it all into our own hands.

Extend the qshader autotest accordingly.

Task-number: QTBUG-81298
Change-Id: If0047b659bd6601ca47b5bbbce1b719630cde01e
Reviewed-by: Paul Olav Tvete <paul.tvete@qt.io>

5 files changed, 251 insertions, 5 deletions

diff --git a/src/gui/rhi/qshader.cpp b/src/gui/rhi/qshader.cpp
index 9203d63cd2..69f4a68215 100644
--- a/src/gui/rhi/qshader.cpp
+++ b/src/gui/rhi/qshader.cpp
@@ -367,7 +367,7 @@ QByteArray QShader::serialized() const
 
     ds << QShaderPrivate::QSB_VERSION;
     ds << int(d->stage);
-    ds << d->desc.toCbor();
+    d->desc.serialize(&ds);
     ds << d->shaders.count();
     for (auto it = d->shaders.cbegin(), itEnd = d->shaders.cend(); it != itEnd; ++it) {
         const QShaderKey &k(it.key());
@@ -428,6 +428,7 @@ QShader QShader::fromSerialized(const QByteArray &data)
     ds >> intVal;
     d->qsbVersion = intVal;
     if (d->qsbVersion != QShaderPrivate::QSB_VERSION
+            && d->qsbVersion != QShaderPrivate::QSB_VERSION_WITH_CBOR
             && d->qsbVersion != QShaderPrivate::QSB_VERSION_WITH_BINARY_JSON
             && d->qsbVersion != QShaderPrivate::QSB_VERSION_WITHOUT_BINDINGS)
     {
@@ -437,14 +438,18 @@ QShader QShader::fromSerialized(const QByteArray &data)
 
     ds >> intVal;
     d->stage = Stage(intVal);
-    QByteArray descBin;
-    ds >> descBin;
-    if (d->qsbVersion > QShaderPrivate::QSB_VERSION_WITH_BINARY_JSON) {
+    if (d->qsbVersion > QShaderPrivate::QSB_VERSION_WITH_CBOR) {
+        d->desc = QShaderDescription::deserialize(&ds);
+    } else if (d->qsbVersion > QShaderPrivate::QSB_VERSION_WITH_BINARY_JSON) {
+        QByteArray descBin;
+        ds >> descBin;
         d->desc = QShaderDescription::fromCbor(descBin);
     } else {
 #if QT_CONFIG(binaryjson) && QT_DEPRECATED_SINCE(5, 15)
         QT_WARNING_PUSH
         QT_WARNING_DISABLE_DEPRECATED
+        QByteArray descBin;
+        ds >> descBin;
         d->desc = QShaderDescription::fromBinaryJson(descBin);
         QT_WARNING_POP
 #else
diff --git a/src/gui/rhi/qshader_p_p.h b/src/gui/rhi/qshader_p_p.h
index 8c89f2b45f..66ef18f391 100644
--- a/src/gui/rhi/qshader_p_p.h
+++ b/src/gui/rhi/qshader_p_p.h
@@ -57,7 +57,8 @@ QT_BEGIN_NAMESPACE
 
 struct Q_GUI_EXPORT QShaderPrivate
 {
-    static const int QSB_VERSION = 3;
+    static const int QSB_VERSION = 4;
+    static const int QSB_VERSION_WITH_CBOR = 3;
     static const int QSB_VERSION_WITH_BINARY_JSON = 2;
     static const int QSB_VERSION_WITHOUT_BINDINGS = 1;
 
diff --git a/src/gui/rhi/qshaderdescription.cpp b/src/gui/rhi/qshaderdescription.cpp
index 76c5d0ebef..96c8d082fc 100644
--- a/src/gui/rhi/qshaderdescription.cpp
+++ b/src/gui/rhi/qshaderdescription.cpp
@@ -36,6 +36,7 @@
 
 #include "qshaderdescription_p_p.h"
 #include <QDebug>
+#include <QDataStream>
 #include <QJsonObject>
 #include <QJsonArray>
 #include <QCborValue>
@@ -358,6 +359,11 @@ QByteArray QShaderDescription::toJson() const
     return d->makeDoc().toJson();
 }
 
+void QShaderDescription::serialize(QDataStream *stream) const
+{
+    d->writeToStream(stream);
+}
+
 #if QT_CONFIG(binaryjson) && QT_DEPRECATED_SINCE(5, 15)
 /*!
     \deprecated
@@ -396,6 +402,13 @@ QShaderDescription QShaderDescription::fromCbor(const QByteArray &data)
     return desc;
 }
 
+QShaderDescription QShaderDescription::deserialize(QDataStream *stream)
+{
+    QShaderDescription desc;
+    QShaderDescriptionPrivate::get(&desc)->loadFromStream(stream);
+    return desc;
+}
+
 /*!
     \return the list of input variables. This includes vertex inputs (sometimes
     called attributes) for the vertex stage, and inputs for other stages
@@ -867,6 +880,15 @@ static void addDeco(QJsonObject *obj, const QShaderDescription::InOutVariable &v
         (*obj)[imageFlagsKey] = int(v.imageFlags);
 }
 
+static void serializeDecorations(QDataStream *stream, const QShaderDescription::InOutVariable &v)
+{
+    (*stream) << v.location;
+    (*stream) << v.binding;
+    (*stream) << v.descriptorSet;
+    (*stream) << int(v.imageFormat);
+    (*stream) << int(v.imageFlags);
+}
+
 static QJsonObject inOutObject(const QShaderDescription::InOutVariable &v)
 {
     QJsonObject obj;
@@ -876,6 +898,13 @@ static QJsonObject inOutObject(const QShaderDescription::InOutVariable &v)
     return obj;
 }
 
+static void serializeInOutVar(QDataStream *stream, const QShaderDescription::InOutVariable &v)
+{
+    (*stream) << v.name;
+    (*stream) << int(v.type);
+    serializeDecorations(stream, v);
+}
+
 static QJsonObject blockMemberObject(const QShaderDescription::BlockVariable &v)
 {
     QJsonObject obj;
@@ -904,6 +933,23 @@ static QJsonObject blockMemberObject(const QShaderDescription::BlockVariable &v)
     return obj;
 }
 
+static void serializeBlockMemberVar(QDataStream *stream, const QShaderDescription::BlockVariable &v)
+{
+    (*stream) << v.name;
+    (*stream) << int(v.type);
+    (*stream) << v.offset;
+    (*stream) << v.size;
+    (*stream) << v.arrayDims.count();
+    for (int dim : v.arrayDims)
+        (*stream) << dim;
+    (*stream) << v.arrayStride;
+    (*stream) << v.matrixStride;
+    (*stream) << v.matrixIsRowMajor;
+    (*stream) << v.structMembers.count();
+    for (const QShaderDescription::BlockVariable &sv : v.structMembers)
+        serializeBlockMemberVar(stream, sv);
+}
+
 QJsonDocument QShaderDescriptionPrivate::makeDoc()
 {
     QJsonObject root;
@@ -1002,6 +1048,67 @@ QJsonDocument QShaderDescriptionPrivate::makeDoc()
     return QJsonDocument(root);
 }
 
+void QShaderDescriptionPrivate::writeToStream(QDataStream *stream)
+{
+    (*stream) << inVars.count();
+    for (const QShaderDescription::InOutVariable &v : qAsConst(inVars))
+        serializeInOutVar(stream, v);
+
+    (*stream) << outVars.count();
+    for (const QShaderDescription::InOutVariable &v : qAsConst(outVars))
+        serializeInOutVar(stream, v);
+
+    (*stream) << uniformBlocks.count();
+    for (const QShaderDescription::UniformBlock &b : uniformBlocks) {
+        (*stream) << b.blockName;
+        (*stream) << b.structName;
+        (*stream) << b.size;
+        (*stream) << b.binding;
+        (*stream) << b.descriptorSet;
+        (*stream) << b.members.count();
+        for (const QShaderDescription::BlockVariable &v : b.members)
+            serializeBlockMemberVar(stream, v);
+    }
+
+    (*stream) << pushConstantBlocks.count();
+    for (const QShaderDescription::PushConstantBlock &b : pushConstantBlocks) {
+        (*stream) << b.name;
+        (*stream) << b.size;
+        (*stream) << b.members.count();
+        for (const QShaderDescription::BlockVariable &v : b.members)
+            serializeBlockMemberVar(stream, v);
+    }
+
+    (*stream) << storageBlocks.count();
+    for (const QShaderDescription::StorageBlock &b : storageBlocks) {
+        (*stream) << b.blockName;
+        (*stream) << b.instanceName;
+        (*stream) << b.knownSize;
+        (*stream) << b.binding;
+        (*stream) << b.descriptorSet;
+        (*stream) << b.members.count();
+        for (const QShaderDescription::BlockVariable &v : b.members)
+            serializeBlockMemberVar(stream, v);
+    }
+
+    (*stream) << combinedImageSamplers.count();
+    for (const QShaderDescription::InOutVariable &v : qAsConst(combinedImageSamplers)) {
+        (*stream) << v.name;
+        (*stream) << int(v.type);
+        serializeDecorations(stream, v);
+    }
+
+    (*stream) << storageImages.count();
+    for (const QShaderDescription::InOutVariable &v : qAsConst(storageImages)) {
+        (*stream) << v.name;
+        (*stream) << int(v.type);
+        serializeDecorations(stream, v);
+    }
+
+    for (size_t i = 0; i < 3; ++i)
+        (*stream) << localSize[i];
+}
+
 static QShaderDescription::InOutVariable inOutVar(const QJsonObject &obj)
 {
     QShaderDescription::InOutVariable var;
@@ -1020,6 +1127,29 @@ static QShaderDescription::InOutVariable inOutVar(const QJsonObject &obj)
     return var;
 }
 
+static void deserializeDecorations(QDataStream *stream, QShaderDescription::InOutVariable *v)
+{
+    (*stream) >> v->location;
+    (*stream) >> v->binding;
+    (*stream) >> v->descriptorSet;
+    int f;
+    (*stream) >> f;
+    v->imageFormat = QShaderDescription::ImageFormat(f);
+    (*stream) >> f;
+    v->imageFlags = QShaderDescription::ImageFlags(f);
+}
+
+static QShaderDescription::InOutVariable deserializeInOutVar(QDataStream *stream)
+{
+    QShaderDescription::InOutVariable var;
+    (*stream) >> var.name;
+    int t;
+    (*stream) >> t;
+    var.type = QShaderDescription::VariableType(t);
+    deserializeDecorations(stream, &var);
+    return var;
+}
+
 static QShaderDescription::BlockVariable blockVar(const QJsonObject &obj)
 {
     QShaderDescription::BlockVariable var;
@@ -1046,6 +1176,30 @@ static QShaderDescription::BlockVariable blockVar(const QJsonObject &obj)
     return var;
 }
 
+static QShaderDescription::BlockVariable deserializeBlockMemberVar(QDataStream *stream)
+{
+    QShaderDescription::BlockVariable var;
+    (*stream) >> var.name;
+    int t;
+    (*stream) >> t;
+    var.type = QShaderDescription::VariableType(t);
+    (*stream) >> var.offset;
+    (*stream) >> var.size;
+    int count;
+    (*stream) >> count;
+    var.arrayDims.resize(count);
+    for (int i = 0; i < count; ++i)
+        (*stream) >> var.arrayDims[i];
+    (*stream) >> var.arrayStride;
+    (*stream) >> var.matrixStride;
+    (*stream) >> var.matrixIsRowMajor;
+    (*stream) >> count;
+    var.structMembers.resize(count);
+    for (int i = 0; i < count; ++i)
+        var.structMembers[i] = deserializeBlockMemberVar(stream);
+    return var;
+}
+
 void QShaderDescriptionPrivate::loadDoc(const QJsonDocument &doc)
 {
     if (doc.isNull()) {
@@ -1150,6 +1304,87 @@ void QShaderDescriptionPrivate::loadDoc(const QJsonDocument &doc)
     }
 }
 
+void QShaderDescriptionPrivate::loadFromStream(QDataStream *stream)
+{
+    Q_ASSERT(ref.loadRelaxed() == 1); // must be detached
+
+    int count;
+    (*stream) >> count;
+    inVars.resize(count);
+    for (int i = 0; i < count; ++i)
+        inVars[i] = deserializeInOutVar(stream);
+
+    (*stream) >> count;
+    outVars.resize(count);
+    for (int i = 0; i < count; ++i)
+        outVars[i] = deserializeInOutVar(stream);
+
+    (*stream) >> count;
+    uniformBlocks.resize(count);
+    for (int i = 0; i < count; ++i) {
+        (*stream) >> uniformBlocks[i].blockName;
+        (*stream) >> uniformBlocks[i].structName;
+        (*stream) >> uniformBlocks[i].size;
+        (*stream) >> uniformBlocks[i].binding;
+        (*stream) >> uniformBlocks[i].descriptorSet;
+        int memberCount;
+        (*stream) >> memberCount;
+        uniformBlocks[i].members.resize(memberCount);
+        for (int memberIdx = 0; memberIdx < memberCount; ++memberIdx)
+            uniformBlocks[i].members[memberIdx] = deserializeBlockMemberVar(stream);
+    }
+
+    (*stream) >> count;
+    pushConstantBlocks.resize(count);
+    for (int i = 0; i < count; ++i) {
+        (*stream) >> pushConstantBlocks[i].name;
+        (*stream) >> pushConstantBlocks[i].size;
+        int memberCount;
+        (*stream) >> memberCount;
+        pushConstantBlocks[i].members.resize(memberCount);
+        for (int memberIdx = 0; memberIdx < memberCount; ++memberIdx)
+            pushConstantBlocks[i].members[memberIdx] = deserializeBlockMemberVar(stream);
+    }
+
+    (*stream) >> count;
+    storageBlocks.resize(count);
+    for (int i = 0; i < count; ++i) {
+        (*stream) >> storageBlocks[i].blockName;
+        (*stream) >> storageBlocks[i].instanceName;
+        (*stream) >> storageBlocks[i].knownSize;
+        (*stream) >> storageBlocks[i].binding;
+        (*stream) >> storageBlocks[i].descriptorSet;
+        int memberCount;
+        (*stream) >> memberCount;
+        storageBlocks[i].members.resize(memberCount);
+        for (int memberIdx = 0; memberIdx < memberCount; ++memberIdx)
+            storageBlocks[i].members[memberIdx] = deserializeBlockMemberVar(stream);
+    }
+
+    (*stream) >> count;
+    combinedImageSamplers.resize(count);
+    for (int i = 0; i < count; ++i) {
+        (*stream) >> combinedImageSamplers[i].name;
+        int t;
+        (*stream) >> t;
+        combinedImageSamplers[i].type = QShaderDescription::VariableType(t);
+        deserializeDecorations(stream, &combinedImageSamplers[i]);
+    }
+
+    (*stream) >> count;
+    storageImages.resize(count);
+    for (int i = 0; i < count; ++i) {
+        (*stream) >> storageImages[i].name;
+        int t;
+        (*stream) >> t;
+        storageImages[i].type = QShaderDescription::VariableType(t);
+        deserializeDecorations(stream, &storageImages[i]);
+    }
+
+    for (size_t i = 0; i < 3; ++i)
+        (*stream) >> localSize[i];
+}
+
 /*!
     Returns \c true if the two QShaderDescription objects \a lhs and \a rhs are
     equal.
diff --git a/src/gui/rhi/qshaderdescription_p.h b/src/gui/rhi/qshaderdescription_p.h
index 872ee8b138..108fc32a56 100644
--- a/src/gui/rhi/qshaderdescription_p.h
+++ b/src/gui/rhi/qshaderdescription_p.h
@@ -56,6 +56,7 @@
 QT_BEGIN_NAMESPACE
 
 struct QShaderDescriptionPrivate;
+class QDataStream;
 
 class Q_GUI_EXPORT QShaderDescription
 {
@@ -69,6 +70,7 @@ public:
     bool isValid() const;
 
     QByteArray toCbor() const;
+    void serialize(QDataStream *stream) const;
     QByteArray toJson() const;
 
 #if QT_CONFIG(binaryjson) && QT_DEPRECATED_SINCE(5, 15)
@@ -76,6 +78,7 @@ public:
     static QShaderDescription fromBinaryJson(const QByteArray &data);
 #endif
     static QShaderDescription fromCbor(const QByteArray &data);
+    static QShaderDescription deserialize(QDataStream *stream);
 
     enum VariableType {
         Unknown = 0,
diff --git a/src/gui/rhi/qshaderdescription_p_p.h b/src/gui/rhi/qshaderdescription_p_p.h
index 1caee24984..69b6e811a1 100644
--- a/src/gui/rhi/qshaderdescription_p_p.h
+++ b/src/gui/rhi/qshaderdescription_p_p.h
@@ -80,7 +80,9 @@ struct Q_GUI_EXPORT QShaderDescriptionPrivate
     static const QShaderDescriptionPrivate *get(const QShaderDescription *desc) { return desc->d; }
 
     QJsonDocument makeDoc();
+    void writeToStream(QDataStream *stream);
     void loadDoc(const QJsonDocument &doc);
+    void loadFromStream(QDataStream *stream);
 
     QAtomicInt ref;
     QVector<QShaderDescription::InOutVariable> inVars;