diff options
Diffstat (limited to 'src/corelib/io/qresource.cpp')
| -rw-r--r-- | src/corelib/io/qresource.cpp | 308 |
1 files changed, 213 insertions, 95 deletions
diff --git a/src/corelib/io/qresource.cpp b/src/corelib/io/qresource.cpp index 564a3e5f51..1077e31797 100644 --- a/src/corelib/io/qresource.cpp +++ b/src/corelib/io/qresource.cpp @@ -1,6 +1,7 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2018 The Qt Company Ltd. +** Copyright (C) 2018 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -56,16 +57,46 @@ #include "private/qabstractfileengine_p.h" #include "private/qnumeric_p.h" #include "private/qsimd_p.h" +#include "private/qtools_p.h" #include "private/qsystemerror_p.h" +#if QT_CONFIG(zstd) +# include <zstd.h> +#endif + #ifdef Q_OS_UNIX # include "private/qcore_unix_p.h" #endif +#if defined(Q_OS_UNIX) && !defined(Q_OS_NACL) && !defined(Q_OS_INTEGRITY) +# define QT_USE_MMAP +# include <sys/mman.h> +#endif + //#define DEBUG_RESOURCE_MATCH QT_BEGIN_NAMESPACE +// Symbols used by code generated by RCC. +// They cause compilation errors if the RCC content couldn't +// be interpreted by this QtCore version. +#if defined(__ELF__) || defined(__APPLE__) // same as RCC generates +# define RCC_FEATURE_SYMBOL(feature) \ + extern Q_CORE_EXPORT const quint8 qt_resourceFeature ## feature; \ + const quint8 qt_resourceFeature ## feature = 0; +#else +# define RCC_FEATURE_SYMBOL(feature) \ + Q_CORE_EXPORT quint8 qResourceFeature ## feature() { return 0; } +#endif + +#ifndef QT_NO_COMPRESS +RCC_FEATURE_SYMBOL(Zlib) +#endif +#if QT_CONFIG(zstd) +RCC_FEATURE_SYMBOL(Zstd) +#endif + +#undef RCC_FEATURE_SYMBOL class QStringSplitter { @@ -98,11 +129,15 @@ public: //resource glue class QResourceRoot { +public: enum Flags { + // must match rcc.h Compressed = 0x01, - Directory = 0x02 + Directory = 0x02, + CompressedZstd = 0x04 }; +private: const uchar *tree, *names, *payloads; int version; inline int findOffset(int node) const { return node * (14 + (version >= 0x02 ? 8 : 0)); } //sizeof each tree element @@ -117,7 +152,15 @@ public: virtual ~QResourceRoot() { } int findNode(const QString &path, const QLocale &locale=QLocale()) const; inline bool isContainer(int node) const { return flags(node) & Directory; } - inline bool isCompressed(int node) const { return flags(node) & Compressed; } + QResource::Compression compressionAlgo(int node) + { + uint compressionFlags = flags(node) & (Compressed | CompressedZstd); + if (compressionFlags == Compressed) + return QResource::ZlibCompression; + if (compressionFlags == CompressedZstd) + return QResource::ZstdCompression; + return QResource::NoCompression; + } const uchar *data(int node, qint64 *size) const; quint64 lastModified(int node) const; QStringList children(int node) const; @@ -229,6 +272,23 @@ static inline QStringList *resourceSearchPaths() \sa {The Qt Resource System}, QFile, QDir, QFileInfo */ +/*! + \enum QResource::Compression + \since 5.13 + + This enum is used by compressionAlgorithm() to indicate which algorithm the + RCC tool used to compress the payload. + + \value NoCompression Contents are not compressed (isCompressed() is false). + \value ZlibCompression Contents are compressed using \l{zlib}{https://zlib.net} and can + be decompressed using the qUncompress() function. + \value ZstdCompression Contents are compressed using \l{zstd}{https://zstd.net}. To + decompress, use the \c{ZSTD_decompress} function from the zstd + library. + + \sa compressionAlgorithm(), isCopressed() +*/ + class QResourcePrivate { public: inline QResourcePrivate(QResource *_q) : q_ptr(_q) { clear(); } @@ -243,12 +303,13 @@ public: QLocale locale; QString fileName, absoluteFilePath; QList<QResourceRoot*> related; - uint container : 1; - mutable uint compressed : 1; mutable qint64 size; + mutable quint64 lastModified; mutable const uchar *data; mutable QStringList children; - mutable quint64 lastModified; + mutable quint8 compressionAlgo; + bool container; + /* 2 or 6 padding bytes */ QResource *q_ptr; Q_DECLARE_PUBLIC(QResource) @@ -258,7 +319,7 @@ void QResourcePrivate::clear() { absoluteFilePath.clear(); - compressed = 0; + compressionAlgo = QResource::NoCompression; data = 0; size = 0; children.clear(); @@ -287,11 +348,11 @@ QResourcePrivate::load(const QString &file) container = res->isContainer(node); if(!container) { data = res->data(node, &size); - compressed = res->isCompressed(node); + compressionAlgo = res->compressionAlgo(node); } else { - data = 0; + data = nullptr; size = 0; - compressed = 0; + compressionAlgo = QResource::NoCompression; } lastModified = res->lastModified(node); } else if(res->isContainer(node) != container) { @@ -301,9 +362,9 @@ QResourcePrivate::load(const QString &file) related.append(res); } else if(res->mappingRootSubdir(file)) { container = true; - data = 0; + data = nullptr; size = 0; - compressed = 0; + compressionAlgo = QResource::NoCompression; lastModified = 0; res->ref.ref(); related.append(res); @@ -493,16 +554,41 @@ bool QResource::isValid() const /*! Returns \c true if the resource represents a file and the data backing it - is in a compressed format, false otherwise. + is in a compressed format, false otherwise. If the data is compressed, + check compressionAlgorithm() to verify what algorithm to use to decompress + the data. - \sa data(), isFile() + \sa data(), compressionType(), isFile() */ bool QResource::isCompressed() const { + return compressionAlgorithm() != NoCompression; +} + +/*! + \since 5.13 + + Returns the compression type that this resource is compressed with, if any. + If it is not compressed, this function returns QResource::NoCompression. + + If this function returns QResource::ZlibCompression, you may decompress the + data using the qUncompress() function. Up until Qt 5.13, this was the only + possible compression algorithm. + + If this function returns QResource::ZstdCompression, you need to use the + Zstandard library functios (\c{<zstd.h> header). Qt does not provide a + wrapper. + + See \l{http://facebook.github.io/zstd/zstd_manual.html}{Zstandard manual}. + + \sa isCompressed(), data(), isFile() +*/ +QResource::Compression QResource::compressionAlgorithm() const +{ Q_D(const QResource); d->ensureInitialized(); - return d->compressed; + return Compression(d->compressionAlgo); } /*! @@ -522,7 +608,7 @@ qint64 QResource::size() const Returns direct access to a read only segment of data that this resource represents. If the resource is compressed the data returns is compressed and qUncompress() must be used to access the data. If the - resource is a directory 0 is returned. + resource is a directory \nullptr is returned. \sa size(), isCompressed(), isFile() */ @@ -573,6 +659,7 @@ QStringList QResource::children() const return d->children; } +#if QT_DEPRECATED_SINCE(5, 13) /*! \obsolete @@ -614,6 +701,7 @@ QResource::searchPaths() QMutexLocker lock(resourceMutex()); return *resourceSearchPaths(); } +#endif inline uint QResourceRoot::hash(int node) const { @@ -853,7 +941,7 @@ Q_CORE_EXPORT bool qRegisterResourceData(int version, const unsigned char *tree, const unsigned char *name, const unsigned char *data) { QMutexLocker lock(resourceMutex()); - if ((version == 0x01 || version == 0x2) && resourceList()) { + if (version >= 0x01 && version <= 0x3 && resourceList()) { bool found = false; QResourceRoot res(version, tree, name, data); for(int i = 0; i < resourceList()->size(); ++i) { @@ -879,7 +967,7 @@ Q_CORE_EXPORT bool qUnregisterResourceData(int version, const unsigned char *tre return false; QMutexLocker lock(resourceMutex()); - if ((version == 0x01 || version == 0x02) && resourceList()) { + if (version >= 0x01 && version <= 0x3 && resourceList()) { QResourceRoot res(version, tree, name, data); for(int i = 0; i < resourceList()->size(); ) { if(*resourceList()->at(i) == res) { @@ -906,11 +994,11 @@ public: inline QDynamicBufferResourceRoot(const QString &_root) : root(_root), buffer(0) { } inline ~QDynamicBufferResourceRoot() { } inline const uchar *mappingBuffer() const { return buffer; } - virtual QString mappingRoot() const override { return root; } - virtual ResourceRootType type() const override { return Resource_Buffer; } + QString mappingRoot() const override { return root; } + ResourceRootType type() const override { return Resource_Buffer; } // size == -1 means "unknown" - bool registerSelf(const uchar *b, int size) + bool registerSelf(const uchar *b, qsizetype size) { // 5 int "pointers" if (size >= 0 && size < 20) @@ -938,11 +1026,27 @@ public: const int name_offset = qFromBigEndian<qint32>(b + offset); offset += 4; + quint32 file_flags = 0; + if (version >= 3) { + file_flags = qFromBigEndian<qint32>(b + offset); + offset += 4; + } + // Some sanity checking for sizes. This is _not_ a security measure. if (size >= 0 && (tree_offset >= size || data_offset >= size || name_offset >= size)) return false; - if (version == 0x01 || version == 0x02) { + // And some sanity checking for features + quint32 acceptableFlags = 0; +#ifndef QT_NO_COMPRESS + acceptableFlags |= Compressed; +#endif + if (QT_CONFIG(zstd)) + acceptableFlags |= CompressedZstd; + if (file_flags & ~acceptableFlags) + return false; + + if (version >= 0x01 && version <= 0x03) { buffer = b; setSource(version, b+tree_offset, b+name_offset, b+data_offset); return true; @@ -951,28 +1055,12 @@ public: } }; -#if defined(Q_OS_UNIX) && !defined (Q_OS_NACL) && !defined(Q_OS_INTEGRITY) -#define QT_USE_MMAP -#endif - -// most of the headers below are already included in qplatformdefs.h -// also this lacks Large File support but that's probably irrelevant -#if defined(QT_USE_MMAP) -// for mmap -QT_BEGIN_INCLUDE_NAMESPACE -#include <sys/mman.h> -#include <errno.h> -QT_END_INCLUDE_NAMESPACE -#endif - - - class QDynamicFileResourceRoot: public QDynamicBufferResourceRoot { QString fileName; // for mmap'ed files, this is what needs to be unmapped. uchar *unmapPointer; - unsigned int unmapLength; + qsizetype unmapLength; public: inline QDynamicFileResourceRoot(const QString &_root) : QDynamicBufferResourceRoot(_root), unmapPointer(0), unmapLength(0) { } @@ -989,76 +1077,78 @@ public: } } QString mappingFile() const { return fileName; } - virtual ResourceRootType type() const override { return Resource_File; } - - bool registerSelf(const QString &f) { - bool fromMM = false; - uchar *data = 0; - unsigned int data_len = 0; + ResourceRootType type() const override { return Resource_File; } -#ifdef QT_USE_MMAP + bool registerSelf(const QString &f); +}; #ifndef MAP_FILE -#define MAP_FILE 0 +# define MAP_FILE 0 #endif #ifndef MAP_FAILED -#define MAP_FAILED -1 +# define MAP_FAILED reinterpret_cast<void *>(-1) #endif - int fd = QT_OPEN(QFile::encodeName(f), O_RDONLY, +bool QDynamicFileResourceRoot::registerSelf(const QString &f) +{ + bool fromMM = false; + uchar *data = nullptr; + qsizetype data_len = 0; + +#ifdef QT_USE_MMAP + int fd = QT_OPEN(QFile::encodeName(f), O_RDONLY, #if defined(Q_OS_WIN) - _S_IREAD | _S_IWRITE + _S_IREAD | _S_IWRITE #else - 0666 + 0666 #endif - ); - if (fd >= 0) { - QT_STATBUF st; - if (!QT_FSTAT(fd, &st)) { - uchar *ptr; - ptr = reinterpret_cast<uchar *>( - mmap(0, st.st_size, // any address, whole file - PROT_READ, // read-only memory - MAP_FILE | MAP_PRIVATE, // swap-backed map from file - fd, 0)); // from offset 0 of fd - if (ptr && ptr != reinterpret_cast<uchar *>(MAP_FAILED)) { - data = ptr; - data_len = st.st_size; - fromMM = true; - } + ); + if (fd >= 0) { + QT_STATBUF st; + if (!QT_FSTAT(fd, &st) && st.st_size <= std::numeric_limits<qsizetype>::max()) { + int protection = PROT_READ; // read-only memory + int flags = MAP_FILE | MAP_PRIVATE; // swap-backed map from file + void *ptr = QT_MMAP(nullptr, st.st_size, // any address, whole file + protection, flags, + fd, 0); // from offset 0 of fd + if (ptr != MAP_FAILED) { + data = static_cast<uchar *>(ptr); + data_len = st.st_size; + fromMM = true; } - ::close(fd); } + QT_CLOSE(fd); + } #endif // QT_USE_MMAP - if(!data) { - QFile file(f); - if (!file.exists()) - return false; - data_len = file.size(); - data = new uchar[data_len]; - - bool ok = false; - if (file.open(QIODevice::ReadOnly)) - ok = (data_len == (uint)file.read((char*)data, data_len)); - if (!ok) { - delete [] data; - data = 0; - data_len = 0; - return false; + if (!data) { + QFile file(f); + bool ok = false; + if (file.open(QIODevice::ReadOnly)) { + qint64 fsize = file.size(); + if (fsize <= std::numeric_limits<qsizetype>::max()) { + data_len = file.size(); + data = new uchar[data_len]; + ok = (data_len == file.read((char*)data, data_len)); } - fromMM = false; } - if (data && QDynamicBufferResourceRoot::registerSelf(data, data_len)) { - if(fromMM) { - unmapPointer = data; - unmapLength = data_len; - } - fileName = f; - return true; + if (!ok) { + delete [] data; + data = nullptr; + data_len = 0; + return false; } - return false; + fromMM = false; } -}; + if (data && QDynamicBufferResourceRoot::registerSelf(data, data_len)) { + if (fromMM) { + unmapPointer = data; + unmapLength = data_len; + } + fileName = f; + return true; + } + return false; +} static QString qt_resource_fixResourceRoot(QString r) { if(!r.isEmpty()) { @@ -1531,12 +1621,40 @@ bool QResourceFileEnginePrivate::unmap(uchar *ptr) void QResourceFileEnginePrivate::uncompress() const { - if (resource.isCompressed() && uncompressed.isEmpty() && resource.size()) { + if (uncompressed.isEmpty() && resource.size()) { + quint64 size; + switch (resource.compressionAlgorithm()) { + case QResource::NoCompression: + return; // nothing to do + + case QResource::ZlibCompression: #ifndef QT_NO_COMPRESS - uncompressed = qUncompress(resource.data(), resource.size()); + uncompressed = qUncompress(resource.data(), resource.size()); #else - Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for compression"); + Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zlib compression"); #endif + break; + + case QResource::ZstdCompression: +#if QT_CONFIG(zstd) + size = ZSTD_getFrameContentSize(resource.data(), resource.size()); + if (!ZSTD_isError(size)) { + if (size >= MaxAllocSize) { + qWarning("QResourceFileEngine::open: content bigger than memory (size %lld)", size); + } else { + uncompressed = QByteArray(size, Qt::Uninitialized); + size = ZSTD_decompress(const_cast<char *>(uncompressed.data()), size, + resource.data(), resource.size()); + } + } + if (ZSTD_isError(size)) + qWarning("QResourceFileEngine::open: error decoding: %s", ZSTD_getErrorName(size)); +#else + Q_UNUSED(size); + Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zstd compression"); +#endif + break; + } } } |