From bd828bacb982a51ee21f03487f8390cfc96cc6d3 Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Thu, 20 Jun 2019 14:26:15 -0700 Subject: QResource: Add API to get the decompressed content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ChangeLog][QtCore][QResource] Added uncompressedSize() and uncompressedData(), which will perform any required decompression on the data, prior to returning (unlike data() and size()). Change-Id: Ief874765cd7b43798de3fffd15aa053bc505dcb1 Reviewed-by: Mårten Nordheim --- src/corelib/io/qresource.cpp | 216 +++++++++++++++------ src/corelib/io/qresource.h | 5 +- .../auto/corelib/io/qresourceengine/compressed.qrc | 5 + .../io/qresourceengine/generateResources.sh | 7 + .../io/qresourceengine/qresourceengine_test.pro | 5 +- .../io/qresourceengine/tst_qresourceengine.cpp | 82 +++++++- .../corelib/io/qresourceengine/uncompressed.rcc | Bin 0 -> 16478 bytes tests/auto/corelib/io/qresourceengine/zlib.rcc | Bin 0 -> 137 bytes tests/auto/corelib/io/qresourceengine/zstd.rcc | Bin 0 -> 112 bytes 9 files changed, 253 insertions(+), 67 deletions(-) create mode 100644 tests/auto/corelib/io/qresourceengine/compressed.qrc create mode 100755 tests/auto/corelib/io/qresourceengine/generateResources.sh create mode 100644 tests/auto/corelib/io/qresourceengine/uncompressed.rcc create mode 100644 tests/auto/corelib/io/qresourceengine/zlib.rcc create mode 100644 tests/auto/corelib/io/qresourceengine/zstd.rcc diff --git a/src/corelib/io/qresource.cpp b/src/corelib/io/qresource.cpp index 43df284d50..225ee0a769 100644 --- a/src/corelib/io/qresource.cpp +++ b/src/corelib/io/qresource.cpp @@ -1,7 +1,7 @@ /**************************************************************************** ** -** Copyright (C) 2018 The Qt Company Ltd. -** Copyright (C) 2019 Intel Corporation. +** Copyright (C) 2019 The Qt Company Ltd. +** Copyright (C) 2020 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -60,14 +60,14 @@ #include "private/qtools_p.h" #include "private/qsystemerror_p.h" +#ifndef QT_NO_COMPRESS +# include +# include +#endif #if QT_CONFIG(zstd) # include #endif -#ifdef Q_OS_UNIX -# include "private/qcore_unix_p.h" -#endif - #if defined(Q_OS_UNIX) && !defined(Q_OS_NACL) && !defined(Q_OS_INTEGRITY) # define QT_USE_MMAP # include @@ -296,6 +296,8 @@ public: void ensureInitialized() const; void ensureChildren() const; + qint64 uncompressedSize() const Q_DECL_PURE_FUNCTION; + qsizetype decompress(char *buffer, qsizetype bufferSize) const; bool load(const QString &file); void clear(); @@ -440,6 +442,78 @@ QResourcePrivate::ensureChildren() const } } +qint64 QResourcePrivate::uncompressedSize() const +{ + switch (compressionAlgo) { + case QResource::NoCompression: + return size; + + case QResource::ZlibCompression: +#ifndef QT_NO_COMPRESS + if (size_t(size) >= sizeof(quint32)) + return qFromBigEndian(data); +#else + Q_ASSERT(!"QResource: Qt built without support for Zlib compression"); + Q_UNREACHABLE(); +#endif + break; + + case QResource::ZstdCompression: { +#if QT_CONFIG(zstd) + size_t n = ZSTD_getFrameContentSize(data, size); + return ZSTD_isError(n) ? -1 : qint64(n); +#else + // This should not happen because we've refused to load such resource + Q_ASSERT(!"QResource: Qt built without support for Zstd compression"); + Q_UNREACHABLE(); +#endif + } + + } + return -1; +} + +qsizetype QResourcePrivate::decompress(char *buffer, qsizetype bufferSize) const +{ + Q_ASSERT(data); + + switch (compressionAlgo) { + case QResource::NoCompression: + Q_UNREACHABLE(); + break; + + case QResource::ZlibCompression: { +#ifndef QT_NO_COMPRESS + uLong len = uLong(bufferSize); + int res = ::uncompress(reinterpret_cast(buffer), &len, + data + sizeof(quint32), uLong(size - sizeof(quint32))); + if (res != Z_OK) { + qWarning("QResource: error decompressing zlib content (%d)", res); + return -1; + } + return len; +#else + Q_UNREACHABLE(); +#endif + } + + case QResource::ZstdCompression: { +#if QT_CONFIG(zstd) + size_t usize = ZSTD_decompress(buffer, bufferSize, data, size); + if (ZSTD_isError(usize)) { + qWarning("QResource: error decompressing zstd content: %s", ZSTD_getErrorName(usize)); + return -1; + } + return usize; +#else + Q_UNREACHABLE(); +#endif + } + } + + return -1; +} + /*! Constructs a QResource pointing to \a file. \a locale is used to load a specific localization of a resource data. @@ -600,9 +674,12 @@ QResource::Compression QResource::compressionAlgorithm() const } /*! - Returns the size of the data backing the resource. + Returns the size of the stored data backing the resource. - \sa data(), isFile() + If the resource is compressed, this function returns the size of the + compressed data. See uncompressedSize() for the uncompressed size. + + \sa data(), uncompressedSize(), isFile() */ qint64 QResource::size() const @@ -613,12 +690,29 @@ qint64 QResource::size() const } /*! - Returns direct access to a read only segment of data that this resource - represents. If the resource is compressed the data returned is compressed - and the appropriate library functions must be used to access the data. If - the resource is a directory \nullptr is returned. + \since 5.15 + + Returns the size of the data in this resource. If the data was not + compressed, this function returns the same as size(). If it was, then this + function extracts the size of the original uncompressed data from the + stored stream. - \sa size(), compressionAlgorithm(), isFile() + \sa size(), uncompressedData(), isFile() +*/ +qint64 QResource::uncompressedSize() const +{ + Q_D(const QResource); + d->ensureInitialized(); + return d->uncompressedSize(); +} + +/*! + Returns direct access to a segment of read-only data, that this resource + represents. If the resource is compressed, the data returned is also + compressed. The caller must then decompress the data or use + uncompressedData(). If the resource is a directory, \c nullptr is returned. + + \sa uncompressedData(), size(), isFile() */ const uchar *QResource::data() const @@ -628,6 +722,42 @@ const uchar *QResource::data() const return d->data; } +/*! + \since 5.15 + + Returns the resource data, decompressing it first, if the data was stored + compressed. If the resource is a directory or an error occurs while + decompressing, a null QByteArray is returned. + + \note If the data was compressed, this function will decompress every time + it is called. The result is not cached between calls. + + \sa uncompressedData(), size(), isCompressed(), isFile() +*/ + +QByteArray QResource::uncompressedData() const +{ + Q_D(const QResource); + qint64 n = uncompressedSize(); + if (n < 0) + return QByteArray(); + if (n > std::numeric_limits::max()) { + qWarning("QResource: compressed content does not fit into a QByteArray; use QFile instead"); + return QByteArray(); + } + if (d->compressionAlgo == NoCompression) + return QByteArray::fromRawData(reinterpret_cast(d->data), n); + + // decompress + QByteArray result(n, Qt::Uninitialized); + n = d->decompress(result.data(), n); + if (n < 0) + result.clear(); + else + result.truncate(n); + return result; +} + /*! \since 5.8 @@ -1451,13 +1581,7 @@ bool QResourceFileEngine::link(const QString &) qint64 QResourceFileEngine::size() const { Q_D(const QResourceFileEngine); - if (!d->resource.isValid()) - return 0; - if (d->resource.compressionAlgorithm() != QResource::NoCompression) { - d->uncompress(); - return d->uncompressed.size(); - } - return d->resource.size(); + return d->resource.isValid() ? d->resource.uncompressedSize() : 0; } qint64 QResourceFileEngine::pos() const @@ -1615,12 +1739,7 @@ uchar *QResourceFileEnginePrivate::map(qint64 offset, qint64 size, QFile::Memory Q_Q(QResourceFileEngine); Q_UNUSED(flags); - qint64 max = resource.size(); - if (resource.compressionAlgorithm() != QResource::NoCompression) { - uncompress(); - max = uncompressed.size(); - } - + qint64 max = resource.uncompressedSize(); qint64 end; if (offset < 0 || size <= 0 || !resource.isValid() || add_overflow(offset, size, &end) || end > max) { @@ -1629,8 +1748,12 @@ uchar *QResourceFileEnginePrivate::map(qint64 offset, qint64 size, QFile::Memory } const uchar *address = resource.data(); - if (resource.compressionAlgorithm() != QResource::NoCompression) + if (resource.compressionAlgorithm() != QResource::NoCompression) { + uncompress(); + if (uncompressed.isNull()) + return nullptr; address = reinterpret_cast(uncompressed.constData()); + } return const_cast(address) + offset; } @@ -1643,41 +1766,10 @@ bool QResourceFileEnginePrivate::unmap(uchar *ptr) void QResourceFileEnginePrivate::uncompress() const { - if (uncompressed.isEmpty() && resource.size()) { - quint64 size; - switch (resource.compressionAlgorithm()) { - case QResource::NoCompression: - return; // nothing to do - - case QResource::ZlibCompression: -#ifndef QT_NO_COMPRESS - uncompressed = qUncompress(resource.data(), resource.size()); -#else - Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zlib compression"); -#endif - break; - - case QResource::ZstdCompression: -#if QT_CONFIG(zstd) - size = ZSTD_getFrameContentSize(resource.data(), resource.size()); - if (!ZSTD_isError(size)) { - if (size >= MaxAllocSize) { - qWarning("QResourceFileEngine::open: content bigger than memory (size %lld)", size); - } else { - uncompressed = QByteArray(size, Qt::Uninitialized); - size = ZSTD_decompress(const_cast(uncompressed.data()), size, - resource.data(), resource.size()); - } - } - if (ZSTD_isError(size)) - qWarning("QResourceFileEngine::open: error decoding: %s", ZSTD_getErrorName(size)); -#else - Q_UNUSED(size); - Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zstd compression"); -#endif - break; - } - } + if (resource.compressionAlgorithm() == QResource::NoCompression + || !uncompressed.isEmpty() || resource.size() == 0) + return; // nothing to do + uncompressed = resource.uncompressedData(); } #endif // !defined(QT_BOOTSTRAPPED) diff --git a/src/corelib/io/qresource.h b/src/corelib/io/qresource.h index 5ee8d5d266..52b0d74d29 100644 --- a/src/corelib/io/qresource.h +++ b/src/corelib/io/qresource.h @@ -1,6 +1,7 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2019 The Qt Company Ltd. +** Copyright (C) 2019 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -75,6 +76,8 @@ public: Compression compressionAlgorithm() const; qint64 size() const; const uchar *data() const; + qint64 uncompressedSize() const; + QByteArray uncompressedData() const; QDateTime lastModified() const; #if QT_DEPRECATED_SINCE(5, 13) diff --git a/tests/auto/corelib/io/qresourceengine/compressed.qrc b/tests/auto/corelib/io/qresourceengine/compressed.qrc new file mode 100644 index 0000000000..f7c7cbc20f --- /dev/null +++ b/tests/auto/corelib/io/qresourceengine/compressed.qrc @@ -0,0 +1,5 @@ + + + zero.txt + + diff --git a/tests/auto/corelib/io/qresourceengine/generateResources.sh b/tests/auto/corelib/io/qresourceengine/generateResources.sh new file mode 100755 index 0000000000..9894f6bfb7 --- /dev/null +++ b/tests/auto/corelib/io/qresourceengine/generateResources.sh @@ -0,0 +1,7 @@ +#!/bin/sh +count=`awk '/ZERO_FILE_LEN/ { print $3 }' tst_qresourceengine.cpp` +dd if=/dev/zero of=zero.txt bs=1 count=$count +rcc --binary -o uncompressed.rcc --no-compress compressed.qrc +rcc --binary -o zlib.rcc --compress-algo zlib --compress 9 compressed.qrc +rcc --binary -o zstd.rcc --compress-algo zstd --compress 19 compressed.qrc +rm zero.txt diff --git a/tests/auto/corelib/io/qresourceengine/qresourceengine_test.pro b/tests/auto/corelib/io/qresourceengine/qresourceengine_test.pro index f523116cc9..345e3d13b9 100644 --- a/tests/auto/corelib/io/qresourceengine/qresourceengine_test.pro +++ b/tests/auto/corelib/io/qresourceengine/qresourceengine_test.pro @@ -1,7 +1,7 @@ CONFIG += testcase TARGET = tst_qresourceengine -QT = core testlib +QT = core-private testlib SOURCES = tst_qresourceengine.cpp RESOURCES += testqrc/test.qrc @@ -15,7 +15,8 @@ QMAKE_DISTCLEAN += $${runtime_resource.target} TESTDATA += \ parentdir.txt \ - testqrc/* + testqrc/* \ + *.rcc GENERATED_TESTDATA = $${runtime_resource.target} android:!android-embedded { diff --git a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp index bdbc1c6928..902e6db12a 100644 --- a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp +++ b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp @@ -27,9 +27,10 @@ ** ****************************************************************************/ - #include #include +#include +#include class tst_QResourceEngine: public QObject { @@ -50,6 +51,8 @@ private slots: void checkUnregisterResource_data(); void checkUnregisterResource(); + void compressedResource_data(); + void compressedResource(); void checkStructure_data(); void checkStructure(); void searchPath_data(); @@ -105,6 +108,75 @@ void tst_QResourceEngine::cleanupTestCase() QVERIFY(QResource::unregisterResource(m_runtimeResourceRcc, "/secondary_root/")); } +void tst_QResourceEngine::compressedResource_data() +{ + QTest::addColumn("fileName"); + QTest::addColumn("compressionAlgo"); + QTest::addColumn("supported"); + + QTest::newRow("uncompressed") + << QFINDTESTDATA("uncompressed.rcc") << int(QResource::NoCompression) << true; + QTest::newRow("zlib") + << QFINDTESTDATA("zlib.rcc") << int(QResource::ZlibCompression) << true; + QTest::newRow("zstd") + << QFINDTESTDATA("zstd.rcc") << int(QResource::ZstdCompression) << QT_CONFIG(zstd); +} + +// Note: generateResource.sh parses this line. Make sure it's a simple number. +#define ZERO_FILE_LEN 16384 +// End note +void tst_QResourceEngine::compressedResource() +{ + QFETCH(QString, fileName); + QFETCH(int, compressionAlgo); + QFETCH(bool, supported); + const QByteArray expectedData(ZERO_FILE_LEN, '\0'); + + QVERIFY(!QResource("zero.txt").isValid()); + QCOMPARE(QResource::registerResource(fileName), supported); + if (!supported) + return; + + auto unregister = qScopeGuard([=] { QResource::unregisterResource(fileName); }); + + QResource resource("zero.txt"); + QVERIFY(resource.isValid()); + QVERIFY(resource.size() > 0); + QVERIFY(resource.data()); + QCOMPARE(resource.compressionAlgorithm(), QResource::Compression(compressionAlgo)); + + if (compressionAlgo == QResource::NoCompression) { + QCOMPARE(resource.size(), ZERO_FILE_LEN); + QCOMPARE(memcmp(resource.data(), expectedData.data(), ZERO_FILE_LEN), 0); + + // API guarantees it will be QByteArray::fromRawData: + QCOMPARE(static_cast(resource.uncompressedData().constData()), + static_cast(resource.data())); + } else { + // reasonable expectation: + QVERIFY(resource.size() < ZERO_FILE_LEN); + } + + // using the engine + QFile f(":/zero.txt"); + QVERIFY(f.exists()); + QVERIFY(f.open(QIODevice::ReadOnly)); + + // verify that we can decompress correctly + QCOMPARE(resource.uncompressedSize(), ZERO_FILE_LEN); + QCOMPARE(f.size(), ZERO_FILE_LEN); + + QByteArray data = resource.uncompressedData(); + QCOMPARE(data.size(), expectedData.size()); + QCOMPARE(data, expectedData); + + // decompression through the engine + data = f.readAll(); + QCOMPARE(data.size(), expectedData.size()); + QCOMPARE(data, expectedData); +} + + void tst_QResourceEngine::checkStructure_data() { QTest::addColumn("pathName"); @@ -140,7 +212,13 @@ void tst_QResourceEngine::checkStructure_data() << "parentdir.txt" << "runtime_resource.rcc" #endif - << "search_file.txt") + << "search_file.txt" +#if defined(BUILTIN_TESTDATA) + << "uncompressed.rcc" + << "zlib.rcc" + << "zstd.rcc" +#endif + ) << rootContents << QLocale::c() << qlonglong(0); diff --git a/tests/auto/corelib/io/qresourceengine/uncompressed.rcc b/tests/auto/corelib/io/qresourceengine/uncompressed.rcc new file mode 100644 index 0000000000..4b009f73d5 Binary files /dev/null and b/tests/auto/corelib/io/qresourceengine/uncompressed.rcc differ diff --git a/tests/auto/corelib/io/qresourceengine/zlib.rcc b/tests/auto/corelib/io/qresourceengine/zlib.rcc new file mode 100644 index 0000000000..66f79fa630 Binary files /dev/null and b/tests/auto/corelib/io/qresourceengine/zlib.rcc differ diff --git a/tests/auto/corelib/io/qresourceengine/zstd.rcc b/tests/auto/corelib/io/qresourceengine/zstd.rcc new file mode 100644 index 0000000000..672fa05d14 Binary files /dev/null and b/tests/auto/corelib/io/qresourceengine/zstd.rcc differ -- cgit v1.2.3