From 448e9fdb57f6a7f7c2ad3986231039f21b20f854 Mon Sep 17 00:00:00 2001 From: Marc Mutz Date: Tue, 2 Feb 2016 12:57:21 +0100 Subject: Enable NRVO in QJsonObject::keys() ... for poor compilers (such as GCC). The test (!d) was changed to match what other member functions test for, e.g. toVariantHash(). Change-Id: I85aee0df6e50da3623ad0afce24abb586e0bd1bc Reviewed-by: Lars Knoll --- src/corelib/json/qjsonobject.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'src/corelib/json') diff --git a/src/corelib/json/qjsonobject.cpp b/src/corelib/json/qjsonobject.cpp index 27f937e750..e43d811157 100644 --- a/src/corelib/json/qjsonobject.cpp +++ b/src/corelib/json/qjsonobject.cpp @@ -270,16 +270,14 @@ QVariantHash QJsonObject::toVariantHash() const */ QStringList QJsonObject::keys() const { - if (!d) - return QStringList(); - QStringList keys; - keys.reserve(o->length); - for (uint i = 0; i < o->length; ++i) { - QJsonPrivate::Entry *e = o->entryAt(i); - keys.append(e->key()); + if (o) { + keys.reserve(o->length); + for (uint i = 0; i < o->length; ++i) { + QJsonPrivate::Entry *e = o->entryAt(i); + keys.append(e->key()); + } } - return keys; } -- cgit v1.2.3 From 03f1a69e9cffe919597373471f7609521a465470 Mon Sep 17 00:00:00 2001 From: Lars Knoll Date: Wed, 18 Mar 2015 08:49:39 +0100 Subject: Avoid size overflows when inserting into very large JSON objects QJson has a size limitation for arrays and objects. Make sure we don't go over that size limit and create corrupt objects when inserting data. Change-Id: I45be3caefc282d8041f38acd120b985ed4389b8c Reviewed-by: Oswald Buddenhagen Reviewed-by: Simon Hausmann Reviewed-by: Thiago Macieira --- src/corelib/json/qjson_p.h | 6 +++++- src/corelib/json/qjsonarray.cpp | 31 +++++++++++++++++++++++++------ src/corelib/json/qjsonarray.h | 6 ++++-- src/corelib/json/qjsondocument.cpp | 4 ++-- src/corelib/json/qjsonobject.cpp | 29 ++++++++++++++++++++++------- src/corelib/json/qjsonobject.h | 6 ++++-- 6 files changed, 62 insertions(+), 20 deletions(-) (limited to 'src/corelib/json') diff --git a/src/corelib/json/qjson_p.h b/src/corelib/json/qjson_p.h index 7f5a2d88a1..1767b3e9e6 100644 --- a/src/corelib/json/qjson_p.h +++ b/src/corelib/json/qjson_p.h @@ -788,7 +788,11 @@ public: if (reserve) { if (reserve < 128) reserve = 128; - size = qMax(size + reserve, size *2); + size = qMax(size + reserve, qMin(size *2, (int)Value::MaxSize)); + if (size > Value::MaxSize) { + qWarning("QJson: Document too large to store in data structure"); + return 0; + } } char *raw = (char *)malloc(size); Q_CHECK_PTR(raw); diff --git a/src/corelib/json/qjsonarray.cpp b/src/corelib/json/qjsonarray.cpp index bb33dbde74..e8d54b5b87 100644 --- a/src/corelib/json/qjsonarray.cpp +++ b/src/corelib/json/qjsonarray.cpp @@ -382,7 +382,7 @@ void QJsonArray::removeAt(int i) if (!a || i < 0 || i >= (int)a->length) return; - detach(); + detach2(); a->removeItems(i, 1); ++d->compactionCounter; if (d->compactionCounter > 32u && d->compactionCounter >= unsigned(a->length) / 2u) @@ -442,7 +442,8 @@ void QJsonArray::insert(int i, const QJsonValue &value) bool compressed; int valueSize = QJsonPrivate::Value::requiredStorage(val, &compressed); - detach(valueSize + sizeof(QJsonPrivate::Value)); + if (!detach2(valueSize + sizeof(QJsonPrivate::Value))) + return; if (!a->length) a->tableOffset = sizeof(QJsonPrivate::Array); @@ -492,7 +493,8 @@ void QJsonArray::replace(int i, const QJsonValue &value) bool compressed; int valueSize = QJsonPrivate::Value::requiredStorage(val, &compressed); - detach(valueSize); + if (!detach2(valueSize)) + return; if (!a->length) a->tableOffset = sizeof(QJsonPrivate::Array); @@ -1122,22 +1124,39 @@ bool QJsonArray::operator!=(const QJsonArray &other) const \internal */ void QJsonArray::detach(uint reserve) +{ + Q_UNUSED(reserve) + Q_ASSERT(!reserve); + detach2(0); +} + +/*! + \internal + */ +bool QJsonArray::detach2(uint reserve) { if (!d) { + if (reserve >= QJsonPrivate::Value::MaxSize) { + qWarning("QJson: Document too large to store in data structure"); + return false; + } d = new QJsonPrivate::Data(reserve, QJsonValue::Array); a = static_cast(d->header->root()); d->ref.ref(); - return; + return true; } if (reserve == 0 && d->ref.load() == 1) - return; + return true; QJsonPrivate::Data *x = d->clone(a, reserve); + if (!x) + return false; x->ref.ref(); if (!d->ref.deref()) delete d; d = x; a = static_cast(d->header->root()); + return true; } /*! @@ -1148,7 +1167,7 @@ void QJsonArray::compact() if (!d || !d->compactionCounter) return; - detach(); + detach2(); d->compact(); a = static_cast(d->header->root()); } diff --git a/src/corelib/json/qjsonarray.h b/src/corelib/json/qjsonarray.h index 611e1f4193..0f86cfc988 100644 --- a/src/corelib/json/qjsonarray.h +++ b/src/corelib/json/qjsonarray.h @@ -185,10 +185,10 @@ public: friend class const_iterator; // stl style - inline iterator begin() { detach(); return iterator(this, 0); } + inline iterator begin() { detach2(); return iterator(this, 0); } inline const_iterator begin() const { return const_iterator(this, 0); } inline const_iterator constBegin() const { return const_iterator(this, 0); } - inline iterator end() { detach(); return iterator(this, size()); } + inline iterator end() { detach2(); return iterator(this, size()); } inline const_iterator end() const { return const_iterator(this, size()); } inline const_iterator constEnd() const { return const_iterator(this, size()); } iterator insert(iterator before, const QJsonValue &value) { insert(before.i, value); return before; } @@ -229,7 +229,9 @@ private: QJsonArray(QJsonPrivate::Data *data, QJsonPrivate::Array *array); void initialize(); void compact(); + // ### Qt 6: remove me and merge with detach2 void detach(uint reserve = 0); + bool detach2(uint reserve = 0); QJsonPrivate::Data *d; QJsonPrivate::Array *a; diff --git a/src/corelib/json/qjsondocument.cpp b/src/corelib/json/qjsondocument.cpp index 3ef006d82d..5f8f807cf0 100644 --- a/src/corelib/json/qjsondocument.cpp +++ b/src/corelib/json/qjsondocument.cpp @@ -482,7 +482,7 @@ void QJsonDocument::setObject(const QJsonObject &object) if (d->compactionCounter) o.compact(); else - o.detach(); + o.detach2(); d = o.d; d->ref.ref(); return; @@ -509,7 +509,7 @@ void QJsonDocument::setArray(const QJsonArray &array) if (d->compactionCounter) a.compact(); else - a.detach(); + a.detach2(); d = a.d; d->ref.ref(); return; diff --git a/src/corelib/json/qjsonobject.cpp b/src/corelib/json/qjsonobject.cpp index e43d811157..8b45dd196b 100644 --- a/src/corelib/json/qjsonobject.cpp +++ b/src/corelib/json/qjsonobject.cpp @@ -389,7 +389,8 @@ QJsonObject::iterator QJsonObject::insert(const QString &key, const QJsonValue & int valueOffset = sizeof(QJsonPrivate::Entry) + QJsonPrivate::qStringSize(key, latinKey); int requiredSize = valueOffset + valueSize; - detach(requiredSize + sizeof(QJsonPrivate::offset)); // offset for the new index entry + if (!detach2(requiredSize + sizeof(QJsonPrivate::offset))) // offset for the new index entry + return iterator(); if (!o->length) o->tableOffset = sizeof(QJsonPrivate::Object); @@ -433,7 +434,7 @@ void QJsonObject::remove(const QString &key) if (!keyExists) return; - detach(); + detach2(); o->removeItems(index, 1); ++d->compactionCounter; if (d->compactionCounter > 32u && d->compactionCounter >= unsigned(o->length) / 2u) @@ -460,7 +461,7 @@ QJsonValue QJsonObject::take(const QString &key) return QJsonValue(QJsonValue::Undefined); QJsonValue v(d, o, o->entryAt(index)->value); - detach(); + detach2(); o->removeItems(index, 1); ++d->compactionCounter; if (d->compactionCounter > 32u && d->compactionCounter >= unsigned(o->length) / 2u) @@ -554,7 +555,7 @@ QJsonObject::iterator QJsonObject::find(const QString &key) int index = o ? o->indexOf(key, &keyExists) : 0; if (!keyExists) return end(); - detach(); + detach2(); return iterator(this, index); } @@ -1060,22 +1061,36 @@ QJsonObject::const_iterator QJsonObject::constFind(const QString &key) const \internal */ void QJsonObject::detach(uint reserve) +{ + Q_UNUSED(reserve) + Q_ASSERT(!reserve); + detach2(reserve); +} + +bool QJsonObject::detach2(uint reserve) { if (!d) { + if (reserve >= QJsonPrivate::Value::MaxSize) { + qWarning("QJson: Document too large to store in data structure"); + return false; + } d = new QJsonPrivate::Data(reserve, QJsonValue::Object); o = static_cast(d->header->root()); d->ref.ref(); - return; + return true; } if (reserve == 0 && d->ref.load() == 1) - return; + return true; QJsonPrivate::Data *x = d->clone(o, reserve); + if (!x) + return false; x->ref.ref(); if (!d->ref.deref()) delete d; d = x; o = static_cast(d->header->root()); + return true; } /*! @@ -1086,7 +1101,7 @@ void QJsonObject::compact() if (!d || !d->compactionCounter) return; - detach(); + detach2(); d->compact(); o = static_cast(d->header->root()); } diff --git a/src/corelib/json/qjsonobject.h b/src/corelib/json/qjsonobject.h index 8535da4a6c..6fb82b7165 100644 --- a/src/corelib/json/qjsonobject.h +++ b/src/corelib/json/qjsonobject.h @@ -182,10 +182,10 @@ public: friend class const_iterator; // STL style - inline iterator begin() { detach(); return iterator(this, 0); } + inline iterator begin() { detach2(); return iterator(this, 0); } inline const_iterator begin() const { return const_iterator(this, 0); } inline const_iterator constBegin() const { return const_iterator(this, 0); } - inline iterator end() { detach(); return iterator(this, size()); } + inline iterator end() { detach2(); return iterator(this, size()); } inline const_iterator end() const { return const_iterator(this, size()); } inline const_iterator constEnd() const { return const_iterator(this, size()); } iterator erase(iterator it); @@ -215,7 +215,9 @@ private: QJsonObject(QJsonPrivate::Data *data, QJsonPrivate::Object *object); void initialize(); + // ### Qt 6: remove me and merge with detach2 void detach(uint reserve = 0); + bool detach2(uint reserve = 0); void compact(); QString keyAt(int i) const; -- cgit v1.2.3 From 4889269ff0fb37130b332863e82dd7c19564116c Mon Sep 17 00:00:00 2001 From: Lars Knoll Date: Wed, 18 Mar 2015 08:49:09 +0100 Subject: Fix quadratic behavior when converting from QVariant The old code called insert for each item, leading to constant reallocation of the data structure. Instead rely on the fact that a QVariantMap (as well as the variant list) is sorted, so we can convert to the QJson data structure in one go without lots of reallocations. Task-number: QTBUG-44737 Change-Id: Id2d38d278fb9afa5e062c7353b4d4215bdfb993c Reviewed-by: Thiago Macieira --- src/corelib/json/qjsonarray.cpp | 41 ++++++++++++++++++++++++++++++-- src/corelib/json/qjsonobject.cpp | 51 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 6 deletions(-) (limited to 'src/corelib/json') diff --git a/src/corelib/json/qjsonarray.cpp b/src/corelib/json/qjsonarray.cpp index e8d54b5b87..dc8851e8e7 100644 --- a/src/corelib/json/qjsonarray.cpp +++ b/src/corelib/json/qjsonarray.cpp @@ -256,8 +256,45 @@ QJsonArray QJsonArray::fromStringList(const QStringList &list) QJsonArray QJsonArray::fromVariantList(const QVariantList &list) { QJsonArray array; - for (QVariantList::const_iterator it = list.constBegin(); it != list.constEnd(); ++it) - array.append(QJsonValue::fromVariant(*it)); + if (list.isEmpty()) + return array; + + array.detach2(1024); + + QVector values; + values.resize(list.size()); + QJsonPrivate::Value *valueData = values.data(); + uint currentOffset = sizeof(QJsonPrivate::Base); + + for (int i = 0; i < list.size(); ++i) { + QJsonValue val = QJsonValue::fromVariant(list.at(i)); + + bool latinOrIntValue; + int valueSize = QJsonPrivate::Value::requiredStorage(val, &latinOrIntValue); + + if (!array.detach2(valueSize)) + return QJsonArray(); + + QJsonPrivate::Value *v = valueData + i; + v->type = (val.t == QJsonValue::Undefined ? QJsonValue::Null : val.t); + v->latinOrIntValue = latinOrIntValue; + v->latinKey = false; + v->value = QJsonPrivate::Value::valueToStore(val, currentOffset); + if (valueSize) + QJsonPrivate::Value::copyData(val, (char *)array.a + currentOffset, latinOrIntValue); + + currentOffset += valueSize; + array.a->size = currentOffset; + } + + // write table + array.a->tableOffset = currentOffset; + if (!array.detach2(sizeof(QJsonPrivate::offset)*values.size())) + return QJsonArray(); + memcpy(array.a->table(), values.constData(), values.size()*sizeof(uint)); + array.a->length = values.size(); + array.a->size = currentOffset + sizeof(QJsonPrivate::offset)*values.size(); + return array; } diff --git a/src/corelib/json/qjsonobject.cpp b/src/corelib/json/qjsonobject.cpp index 8b45dd196b..b83c8dd19a 100644 --- a/src/corelib/json/qjsonobject.cpp +++ b/src/corelib/json/qjsonobject.cpp @@ -197,11 +197,54 @@ QJsonObject &QJsonObject::operator =(const QJsonObject &other) */ QJsonObject QJsonObject::fromVariantMap(const QVariantMap &map) { - // ### this is implemented the trivial way, not the most efficient way - QJsonObject object; - for (QVariantMap::const_iterator it = map.constBegin(); it != map.constEnd(); ++it) - object.insert(it.key(), QJsonValue::fromVariant(it.value())); + if (map.isEmpty()) + return object; + + object.detach2(1024); + + QVector offsets; + QJsonPrivate::offset currentOffset; + currentOffset = sizeof(QJsonPrivate::Base); + + // the map is already sorted, so we can simply append one entry after the other and + // write the offset table at the end + for (QVariantMap::const_iterator it = map.constBegin(); it != map.constEnd(); ++it) { + QString key = it.key(); + QJsonValue val = QJsonValue::fromVariant(it.value()); + + bool latinOrIntValue; + int valueSize = QJsonPrivate::Value::requiredStorage(val, &latinOrIntValue); + + bool latinKey = QJsonPrivate::useCompressed(key); + int valueOffset = sizeof(QJsonPrivate::Entry) + QJsonPrivate::qStringSize(key, latinKey); + int requiredSize = valueOffset + valueSize; + + if (!object.detach2(requiredSize + sizeof(QJsonPrivate::offset))) // offset for the new index entry + return QJsonObject(); + + QJsonPrivate::Entry *e = reinterpret_cast(reinterpret_cast(object.o) + currentOffset); + e->value.type = val.t; + e->value.latinKey = latinKey; + e->value.latinOrIntValue = latinOrIntValue; + e->value.value = QJsonPrivate::Value::valueToStore(val, (char *)e - (char *)object.o + valueOffset); + QJsonPrivate::copyString((char *)(e + 1), key, latinKey); + if (valueSize) + QJsonPrivate::Value::copyData(val, (char *)e + valueOffset, latinOrIntValue); + + offsets << currentOffset; + currentOffset += requiredSize; + object.o->size = currentOffset; + } + + // write table + object.o->tableOffset = currentOffset; + if (!object.detach2(sizeof(QJsonPrivate::offset)*offsets.size())) + return QJsonObject(); + memcpy(object.o->table(), offsets.constData(), offsets.size()*sizeof(uint)); + object.o->length = offsets.size(); + object.o->size = currentOffset + sizeof(QJsonPrivate::offset)*offsets.size(); + return object; } -- cgit v1.2.3