diff options
Diffstat (limited to 'src/corelib/io/qresource.cpp')
-rw-r--r-- | src/corelib/io/qresource.cpp | 802 |
1 files changed, 442 insertions, 360 deletions
diff --git a/src/corelib/io/qresource.cpp b/src/corelib/io/qresource.cpp index 0333f62870..581e1e75ef 100644 --- a/src/corelib/io/qresource.cpp +++ b/src/corelib/io/qresource.cpp @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2019 The Qt Company Ltd. -** Copyright (C) 2020 Intel Corporation. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtCore module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2019 The Qt Company Ltd. +// Copyright (C) 2020 Intel Corporation. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qresource.h" #include "qresource_p.h" @@ -69,15 +33,27 @@ # include <zstd.h> #endif -#if defined(Q_OS_UNIX) && !defined(Q_OS_NACL) && !defined(Q_OS_INTEGRITY) +#if defined(Q_OS_UNIX) && !defined(Q_OS_INTEGRITY) # define QT_USE_MMAP # include <sys/mman.h> +# ifdef Q_OS_LINUX +// since 5.7, so define in case we're being compiled with older kernel headers +# define MREMAP_DONTUNMAP 4 +# elif defined(Q_OS_DARWIN) +# include <mach/mach.h> +# include <mach/vm_map.h> +# endif +#endif +#ifdef Q_OS_WIN +# include <qt_windows.h> #endif //#define DEBUG_RESOURCE_MATCH QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; + // Symbols used by code generated by RCC. // They cause compilation errors if the RCC content couldn't // be interpreted by this QtCore version. @@ -99,6 +75,7 @@ RCC_FEATURE_SYMBOL(Zstd) #undef RCC_FEATURE_SYMBOL +namespace { class QStringSplitter { public: @@ -107,14 +84,16 @@ public: { } - inline bool hasNext() { + inline bool hasNext() + { while (m_pos < m_len && m_data[m_pos] == m_splitChar) ++m_pos; return m_pos < m_len; } - inline QStringView next() { - int start = m_pos; + inline QStringView next() + { + const qsizetype start = m_pos; while (m_pos < m_len && m_data[m_pos] != m_splitChar) ++m_pos; return QStringView(m_data + start, m_pos - start); @@ -123,21 +102,20 @@ public: const QChar *m_data; qsizetype m_len; qsizetype m_pos = 0; - QChar m_splitChar = QLatin1Char('/'); + QChar m_splitChar = u'/'; }; - -//resource glue +// resource glue class QResourceRoot { public: - enum Flags - { + enum Flags { // must match rcc.h Compressed = 0x01, Directory = 0x02, CompressedZstd = 0x04 }; + private: const uchar *tree, *names, *payloads; int version; @@ -163,7 +141,7 @@ public: return QResource::NoCompression; } const uchar *data(int node, qint64 *size) const; - quint64 lastModified(int node) const; + qint64 lastModified(int node) const; QStringList children(int node) const; virtual QString mappingRoot() const { return QString(); } bool mappingRootSubdir(const QString &path, QString *match = nullptr) const; @@ -188,20 +166,22 @@ static QString cleanPath(const QString &_path) QString path = QDir::cleanPath(_path); // QDir::cleanPath does not remove two trailing slashes under _Windows_ // due to support for UNC paths. Remove those manually. - if (path.startsWith(QLatin1String("//"))) + if (path.startsWith("//"_L1)) path.remove(0, 1); return path; } +} // unnamed namespace -Q_DECLARE_TYPEINFO(QResourceRoot, Q_MOVABLE_TYPE); +Q_DECLARE_TYPEINFO(QResourceRoot, Q_RELOCATABLE_TYPE); typedef QList<QResourceRoot*> ResourceList; +namespace { struct QResourceGlobalData { QRecursiveMutex resourceMutex; ResourceList resourceList; - QStringList resourceSearchPaths; }; +} Q_GLOBAL_STATIC(QResourceGlobalData, resourceGlobalData) static inline QRecursiveMutex &resourceMutex() @@ -210,9 +190,6 @@ static inline QRecursiveMutex &resourceMutex() static inline ResourceList *resourceList() { return &resourceGlobalData->resourceList; } -static inline QStringList *resourceSearchPaths() -{ return &resourceGlobalData->resourceSearchPaths; } - /*! \class QResource \inmodule QtCore @@ -270,6 +247,19 @@ static inline QStringList *resourceSearchPaths() itself will be unmapped from memory when the last QResource that points to it is destroyed. + \section2 Corruption and Security + + The QResource class performs some checks on the file passed to determine + whether it is supported by the current version of Qt. Those tests are only + to check the file header does not request features (such as Zstandard + decompression) that have not been compiled in or that the file is not of a + future version of Qt. They do not confirm the validity of the entire file. + + QResource should not be used on files whose provenance cannot be trusted. + Applications should be designed to attempt to load only resource files + whose provenance is at least as trustworthy as that of the application + itself or its plugins. + \sa {The Qt Resource System}, QFile, QDir, QFileInfo */ @@ -283,7 +273,7 @@ static inline QStringList *resourceSearchPaths() \value NoCompression Contents are not compressed \value ZlibCompression Contents are compressed using \l{https://zlib.net}{zlib} and can be decompressed using the qUncompress() function. - \value ZstdCompression Contents are compressed using \l{https://zstd.net}{zstd}. To + \value ZstdCompression Contents are compressed using \l{Zstandard Site}{zstd}. To decompress, use the \c{ZSTD_decompress} function from the zstd library. @@ -303,14 +293,16 @@ public: bool load(const QString &file); void clear(); + static bool mayRemapData(const QResource &resource); + QLocale locale; QString fileName, absoluteFilePath; - QList<QResourceRoot*> related; - mutable qint64 size; - mutable quint64 lastModified; - mutable const uchar *data; + QList<QResourceRoot *> related; + qint64 size; + qint64 lastModified; + const uchar *data; mutable QStringList children; - mutable quint8 compressionAlgo; + quint8 compressionAlgo; bool container; /* 2 or 6 padding bytes */ @@ -318,8 +310,7 @@ public: Q_DECLARE_PUBLIC(QResource) }; -void -QResourcePrivate::clear() +void QResourcePrivate::clear() { absoluteFilePath.clear(); compressionAlgo = QResource::NoCompression; @@ -328,28 +319,27 @@ QResourcePrivate::clear() children.clear(); lastModified = 0; container = 0; - for(int i = 0; i < related.size(); ++i) { + for (int i = 0; i < related.size(); ++i) { QResourceRoot *root = related.at(i); - if(!root->ref.deref()) + if (!root->ref.deref()) delete root; } related.clear(); } -bool -QResourcePrivate::load(const QString &file) +bool QResourcePrivate::load(const QString &file) { related.clear(); const auto locker = qt_scoped_lock(resourceMutex()); const ResourceList *list = resourceList(); QString cleaned = cleanPath(file); - for(int i = 0; i < list->size(); ++i) { + for (int i = 0; i < list->size(); ++i) { QResourceRoot *res = list->at(i); const int node = res->findNode(cleaned, locale); - if(node != -1) { - if(related.isEmpty()) { + if (node != -1) { + if (related.isEmpty()) { container = res->isContainer(node); - if(!container) { + if (!container) { data = res->data(node, &size); compressionAlgo = res->compressionAlgo(node); } else { @@ -358,12 +348,13 @@ QResourcePrivate::load(const QString &file) compressionAlgo = QResource::NoCompression; } lastModified = res->lastModified(node); - } else if(res->isContainer(node) != container) { - qWarning("QResourceInfo: Resource [%s] has both data and children!", file.toLatin1().constData()); + } else if (res->isContainer(node) != container) { + qWarning("QResourceInfo: Resource [%s] has both data and children!", + file.toLatin1().constData()); } res->ref.ref(); related.append(res); - } else if(res->mappingRootSubdir(file)) { + } else if (res->mappingRootSubdir(file)) { container = true; data = nullptr; size = 0; @@ -376,61 +367,52 @@ QResourcePrivate::load(const QString &file) return !related.isEmpty(); } -void -QResourcePrivate::ensureInitialized() const +void QResourcePrivate::ensureInitialized() const { - if(!related.isEmpty()) + if (!related.isEmpty()) return; QResourcePrivate *that = const_cast<QResourcePrivate *>(this); - if(fileName == QLatin1String(":")) - that->fileName += QLatin1Char('/'); + if (fileName == ":"_L1) + that->fileName += u'/'; that->absoluteFilePath = fileName; - if(!that->absoluteFilePath.startsWith(QLatin1Char(':'))) - that->absoluteFilePath.prepend(QLatin1Char(':')); + if (!that->absoluteFilePath.startsWith(u':')) + that->absoluteFilePath.prepend(u':'); QStringView path(fileName); - if(path.startsWith(QLatin1Char(':'))) + if (path.startsWith(u':')) path = path.mid(1); - if(path.startsWith(QLatin1Char('/'))) { + if (path.startsWith(u'/')) { that->load(path.toString()); } else { - const auto locker = qt_scoped_lock(resourceMutex()); - QStringList searchPaths = *resourceSearchPaths(); - searchPaths << QLatin1String(""); - for(int i = 0; i < searchPaths.size(); ++i) { - const QString searchPath(searchPaths.at(i) + QLatin1Char('/') + path); - if(that->load(searchPath)) { - that->absoluteFilePath = QLatin1Char(':') + searchPath; - break; - } - } + // Should we search QDir::searchPath() before falling back to root ? + const QString searchPath(u'/' + path); + if (that->load(searchPath)) + that->absoluteFilePath = u':' + searchPath; } } -void -QResourcePrivate::ensureChildren() const +void QResourcePrivate::ensureChildren() const { ensureInitialized(); - if(!children.isEmpty() || !container || related.isEmpty()) + if (!children.isEmpty() || !container || related.isEmpty()) return; QString path = absoluteFilePath, k; - if(path.startsWith(QLatin1Char(':'))) + if (path.startsWith(u':')) path = path.mid(1); - QDuplicateTracker<QString> kids; - kids.reserve(related.size()); + QDuplicateTracker<QString> kids(related.size()); QString cleaned = cleanPath(path); - for(int i = 0; i < related.size(); ++i) { + for (int i = 0; i < related.size(); ++i) { QResourceRoot *res = related.at(i); - if(res->mappingRootSubdir(path, &k) && !k.isEmpty()) { + if (res->mappingRootSubdir(path, &k) && !k.isEmpty()) { if (!kids.hasSeen(k)) children += k; } else { const int node = res->findNode(cleaned); - if(node != -1) { + if (node != -1) { QStringList related_children = res->children(node); - for(int kid = 0; kid < related_children.size(); ++kid) { + for (int kid = 0; kid < related_children.size(); ++kid) { k = related_children.at(kid); if (!kids.hasSeen(k)) children += k; @@ -466,7 +448,6 @@ qint64 QResourcePrivate::uncompressedSize() const Q_UNREACHABLE(); #endif } - } return -1; } @@ -474,6 +455,10 @@ qint64 QResourcePrivate::uncompressedSize() const qsizetype QResourcePrivate::decompress(char *buffer, qsizetype bufferSize) const { Q_ASSERT(data); +#if defined(QT_NO_COMPRESS) && !QT_CONFIG(zstd) + Q_UNUSED(buffer); + Q_UNUSED(bufferSize); +#endif switch (compressionAlgo) { case QResource::NoCompression: @@ -483,8 +468,8 @@ qsizetype QResourcePrivate::decompress(char *buffer, qsizetype bufferSize) const case QResource::ZlibCompression: { #ifndef QT_NO_COMPRESS uLong len = uLong(bufferSize); - int res = ::uncompress(reinterpret_cast<Bytef *>(buffer), &len, - data + sizeof(quint32), uLong(size - sizeof(quint32))); + int res = ::uncompress(reinterpret_cast<Bytef *>(buffer), &len, data + sizeof(quint32), + uLong(size - sizeof(quint32))); if (res != Z_OK) { qWarning("QResource: error decompressing zlib content (%d)", res); return -1; @@ -634,7 +619,7 @@ bool QResource::isValid() const possible compression algorithm. If this function returns QResource::ZstdCompression, you need to use the - Zstandard library functios (\c{<zstd.h> header). Qt does not provide a + Zstandard library functions (\c{<zstd.h>} header). Qt does not provide a wrapper. See \l{http://facebook.github.io/zstd/zstd_manual.html}{Zstandard manual}. @@ -707,7 +692,7 @@ const uchar *QResource::data() const \note If the data was compressed, this function will decompress every time it is called. The result is not cached between calls. - \sa uncompressedSize(), size(), isCompressed(), isFile() + \sa uncompressedSize(), size(), compressionAlgorithm(), isFile() */ QByteArray QResource::uncompressedData() const @@ -776,16 +761,16 @@ QStringList QResource::children() const inline uint QResourceRoot::hash(int node) const { - if(!node) //root + if (!node) // root return 0; const int offset = findOffset(node); qint32 name_offset = qFromBigEndian<qint32>(tree + offset); - name_offset += 2; //jump past name length + name_offset += 2; // jump past name length return qFromBigEndian<quint32>(names + name_offset); } inline QString QResourceRoot::name(int node) const { - if(!node) // root + if (!node) // root return QString(); const int offset = findOffset(node); @@ -793,11 +778,11 @@ inline QString QResourceRoot::name(int node) const qint32 name_offset = qFromBigEndian<qint32>(tree + offset); quint16 name_length = qFromBigEndian<qint16>(names + name_offset); name_offset += 2; - name_offset += 4; //jump past hash + name_offset += 4; // jump past hash ret.resize(name_length); QChar *strData = ret.data(); - qFromBigEndian<ushort>(names + name_offset, name_length, strData); + qFromBigEndian<char16_t>(names + name_offset, name_length, strData); return ret; } @@ -806,31 +791,31 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const QString path = _path; { QString root = mappingRoot(); - if(!root.isEmpty()) { - if(root == path) { - path = QLatin1Char('/'); + if (!root.isEmpty()) { + if (root == path) { + path = u'/'; } else { - if(!root.endsWith(QLatin1Char('/'))) - root += QLatin1Char('/'); - if(path.size() >= root.size() && path.startsWith(root)) - path = path.mid(root.length()-1); - if(path.isEmpty()) - path = QLatin1Char('/'); + if (!root.endsWith(u'/')) + root += u'/'; + if (path.size() >= root.size() && path.startsWith(root)) + path = path.mid(root.size() - 1); + if (path.isEmpty()) + path = u'/'; } } } #ifdef DEBUG_RESOURCE_MATCH - qDebug() << "!!!!" << "START" << path << locale.country() << locale.language(); + qDebug() << "!!!!" << "START" << path << locale.territory() << locale.language(); #endif - if(path == QLatin1String("/")) + if (path == "/"_L1) return 0; - //the root node is always first + // the root node is always first qint32 child_count = qFromBigEndian<qint32>(tree + 6); qint32 child = qFromBigEndian<qint32>(tree + 10); - //now iterate up the tree + // now iterate up the tree int node = -1; QStringSplitter splitter(path); @@ -839,20 +824,20 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const #ifdef DEBUG_RESOURCE_MATCH qDebug() << " CHILDREN" << segment; - for(int j = 0; j < child_count; ++j) { - qDebug() << " " << child+j << " :: " << name(child+j); + for (int j = 0; j < child_count; ++j) { + qDebug() << " " << child + j << " :: " << name(child + j); } #endif const uint h = qt_hash(segment); - //do the binary search for the hash - int l = 0, r = child_count-1; - int sub_node = (l+r+1)/2; - while(r != l) { - const uint sub_node_hash = hash(child+sub_node); - if(h == sub_node_hash) + // do the binary search for the hash + int l = 0, r = child_count - 1; + int sub_node = (l + r + 1) / 2; + while (r != l) { + const uint sub_node_hash = hash(child + sub_node); + if (h == sub_node_hash) break; - else if(h < sub_node_hash) + else if (h < sub_node_hash) r = sub_node - 1; else l = sub_node; @@ -860,26 +845,27 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const } sub_node += child; - //now do the "harder" compares + // now do the "harder" compares bool found = false; - if(hash(sub_node) == h) { - while(sub_node > child && hash(sub_node-1) == h) //backup for collisions + if (hash(sub_node) == h) { + while (sub_node > child && hash(sub_node - 1) == h) // backup for collisions --sub_node; - for(; sub_node < child+child_count && hash(sub_node) == h; ++sub_node) { //here we go... - if(name(sub_node) == segment) { + for (; sub_node < child + child_count && hash(sub_node) == h; + ++sub_node) { // here we go... + if (name(sub_node) == segment) { found = true; int offset = findOffset(sub_node); #ifdef DEBUG_RESOURCE_MATCH qDebug() << " TRY" << sub_node << name(sub_node) << offset; #endif - offset += 4; //jump past name + offset += 4; // jump past name const qint16 flags = qFromBigEndian<qint16>(tree + offset); offset += 2; - if(!splitter.hasNext()) { - if(!(flags & Directory)) { - const qint16 country = qFromBigEndian<qint16>(tree + offset); + if (!splitter.hasNext()) { + if (!(flags & Directory)) { + const qint16 territory = qFromBigEndian<qint16>(tree + offset); offset += 2; const qint16 language = qFromBigEndian<qint16>(tree + offset); @@ -887,13 +873,16 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const #ifdef DEBUG_RESOURCE_MATCH qDebug() << " " << "LOCALE" << country << language; #endif - if(country == locale.country() && language == locale.language()) { + if (territory == locale.territory() && language == locale.language()) { #ifdef DEBUG_RESOURCE_MATCH qDebug() << "!!!!" << "FINISHED" << __LINE__ << sub_node; #endif return sub_node; - } else if((country == QLocale::AnyCountry && language == locale.language()) || - (country == QLocale::AnyCountry && language == QLocale::C && node == -1)) { + } else if ((territory == QLocale::AnyTerritory + && language == locale.language()) + || (territory == QLocale::AnyTerritory + && language == QLocale::C + && node == -1)) { node = sub_node; } continue; @@ -906,7 +895,7 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const } } - if(!(flags & Directory)) + if (!(flags & Directory)) return -1; child_count = qFromBigEndian<qint32>(tree + offset); @@ -916,7 +905,7 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const } } } - if(!found) + if (!found) break; } #ifdef DEBUG_RESOURCE_MATCH @@ -926,28 +915,28 @@ int QResourceRoot::findNode(const QString &_path, const QLocale &locale) const } short QResourceRoot::flags(int node) const { - if(node == -1) + if (node == -1) return 0; - const int offset = findOffset(node) + 4; //jump past name + const int offset = findOffset(node) + 4; // jump past name return qFromBigEndian<qint16>(tree + offset); } const uchar *QResourceRoot::data(int node, qint64 *size) const { - if(node == -1) { + if (node == -1) { *size = 0; return nullptr; } - int offset = findOffset(node) + 4; //jump past name + int offset = findOffset(node) + 4; // jump past name const qint16 flags = qFromBigEndian<qint16>(tree + offset); offset += 2; - offset += 4; //jump past locale + offset += 4; // jump past locale - if(!(flags & Directory)) { + if (!(flags & Directory)) { const qint32 data_offset = qFromBigEndian<qint32>(tree + offset); const quint32 data_length = qFromBigEndian<quint32>(payloads + data_offset); - const uchar *ret = payloads+data_offset+4; + const uchar *ret = payloads + data_offset + 4; *size = data_length; return ret; } @@ -955,32 +944,32 @@ const uchar *QResourceRoot::data(int node, qint64 *size) const return nullptr; } -quint64 QResourceRoot::lastModified(int node) const +qint64 QResourceRoot::lastModified(int node) const { if (node == -1 || version < 0x02) return 0; const int offset = findOffset(node) + 14; - return qFromBigEndian<quint64>(tree + offset); + return qFromBigEndian<qint64>(tree + offset); } QStringList QResourceRoot::children(int node) const { - if(node == -1) + if (node == -1) return QStringList(); - int offset = findOffset(node) + 4; //jump past name + int offset = findOffset(node) + 4; // jump past name const qint16 flags = qFromBigEndian<qint16>(tree + offset); offset += 2; QStringList ret; - if(flags & Directory) { + if (flags & Directory) { const qint32 child_count = qFromBigEndian<qint32>(tree + offset); offset += 4; const qint32 child_off = qFromBigEndian<qint32>(tree + offset); ret.reserve(child_count); - for(int i = child_off; i < child_off+child_count; ++i) + for (int i = child_off; i < child_off + child_count; ++i) ret << name(i); } return ret; @@ -1024,7 +1013,7 @@ Q_CORE_EXPORT bool qRegisterResourceData(int version, const unsigned char *tree, break; } } - if(!found) { + if (!found) { QResourceRoot *root = new QResourceRoot(version, tree, name, data); root->ref.ref(); list->append(root); @@ -1044,10 +1033,10 @@ Q_CORE_EXPORT bool qUnregisterResourceData(int version, const unsigned char *tre if (version >= 0x01 && version <= 0x3) { QResourceRoot res(version, tree, name, data); ResourceList *list = resourceList(); - for (int i = 0; i < list->size(); ) { + for (int i = 0; i < list->size();) { if (*list->at(i) == res) { QResourceRoot *root = list->takeAt(i); - if(!root->ref.deref()) + if (!root->ref.deref()) delete root; } else { ++i; @@ -1058,9 +1047,9 @@ Q_CORE_EXPORT bool qUnregisterResourceData(int version, const unsigned char *tre return false; } -//run time resource creation - -class QDynamicBufferResourceRoot: public QResourceRoot +namespace { +// run time resource creation +class QDynamicBufferResourceRoot : public QResourceRoot { QString root; const uchar *buffer; @@ -1079,12 +1068,12 @@ public: if (size >= 0 && size < 20) return false; - //setup the data now + // setup the data now int offset = 0; - //magic number - if(b[offset+0] != 'q' || b[offset+1] != 'r' || - b[offset+2] != 'e' || b[offset+3] != 's') { + // magic number + if (b[offset + 0] != 'q' || b[offset + 1] != 'r' || b[offset + 2] != 'e' + || b[offset + 3] != 's') { return false; } offset += 4; @@ -1123,15 +1112,20 @@ public: if (version >= 0x01 && version <= 0x03) { buffer = b; - setSource(version, b+tree_offset, b+name_offset, b+data_offset); + setSource(version, b + tree_offset, b + name_offset, b + data_offset); return true; } return false; } }; -class QDynamicFileResourceRoot: public QDynamicBufferResourceRoot +class QDynamicFileResourceRoot : public QDynamicBufferResourceRoot { +public: + static uchar *map_sys(QFile &file, qint64 base, qsizetype size); + static void unmap_sys(void *base, qsizetype size); + +private: QString fileName; // for mmap'ed files, this is what needs to be unmapped. uchar *unmapPointer; @@ -1142,22 +1136,18 @@ public: : QDynamicBufferResourceRoot(_root), unmapPointer(nullptr), unmapLength(0) { } ~QDynamicFileResourceRoot() { -#if defined(QT_USE_MMAP) - if (unmapPointer) { - munmap((char*)unmapPointer, unmapLength); - unmapPointer = nullptr; - unmapLength = 0; - } else -#endif - { - delete [] mappingBuffer(); - } + if (wasMemoryMapped()) + unmap_sys(unmapPointer, unmapLength); + else + delete[] mappingBuffer(); } QString mappingFile() const { return fileName; } ResourceRootType type() const override { return Resource_File; } + bool wasMemoryMapped() const { return unmapPointer; } bool registerSelf(const QString &f); }; +} // unnamed namespace #ifndef MAP_FILE # define MAP_FILE 0 @@ -1166,49 +1156,69 @@ public: # define MAP_FAILED reinterpret_cast<void *>(-1) #endif -bool QDynamicFileResourceRoot::registerSelf(const QString &f) +void QDynamicFileResourceRoot::unmap_sys(void *base, qsizetype size) { - bool fromMM = false; - uchar *data = nullptr; - qsizetype data_len = 0; +#if defined(QT_USE_MMAP) + munmap(base, size); +#elif defined(Q_OS_WIN) + Q_UNUSED(size) + UnmapViewOfFile(reinterpret_cast<void *>(base)); +#endif +} + +// Note: caller must ensure \a offset and \a size are acceptable to the OS. +uchar *QDynamicFileResourceRoot::map_sys(QFile &file, qint64 offset, qsizetype size) +{ + Q_ASSERT(file.isOpen()); + void *ptr = nullptr; + if (size < 0) + size = qMin(file.size() - offset, (std::numeric_limits<qsizetype>::max)()); + // We don't use QFile::map() here because we want to dispose of the QFile object #if defined(QT_USE_MMAP) - int fd = QT_OPEN(QFile::encodeName(f), O_RDONLY, 0666); - if (fd >= 0) { - QT_STATBUF st; - if (!QT_FSTAT(fd, &st) && st.st_size <= std::numeric_limits<qsizetype>::max()) { - int protection = PROT_READ; // read-only memory - int flags = MAP_FILE | MAP_PRIVATE; // swap-backed map from file - void *ptr = QT_MMAP(nullptr, st.st_size, // any address, whole file - protection, flags, - fd, 0); // from offset 0 of fd - if (ptr != MAP_FAILED) { - data = static_cast<uchar *>(ptr); - data_len = st.st_size; - fromMM = true; - } + int fd = file.handle(); + int protection = PROT_READ; // read-only memory + int flags = MAP_FILE | MAP_PRIVATE; // swap-backed map from file + ptr = QT_MMAP(nullptr, size, protection, flags, fd, offset); + if (ptr == MAP_FAILED) + ptr = nullptr; +#elif defined(Q_OS_WIN) + int fd = file.handle(); + HANDLE fileHandle = reinterpret_cast<HANDLE>(_get_osfhandle(fd)); + if (fileHandle != INVALID_HANDLE_VALUE) { + HANDLE mapHandle = CreateFileMapping(fileHandle, 0, PAGE_WRITECOPY, 0, 0, 0); + if (mapHandle) { + ptr = MapViewOfFile(mapHandle, FILE_MAP_COPY, DWORD(offset >> 32), DWORD(offset), size); + CloseHandle(mapHandle); } - QT_CLOSE(fd); } #endif // QT_USE_MMAP - if (!data) { - QFile file(f); + return static_cast<uchar *>(ptr); +} + +bool QDynamicFileResourceRoot::registerSelf(const QString &f) +{ + QFile file(f); + if (!file.open(QIODevice::ReadOnly)) + return false; + + qint64 data_len = file.size(); + if (data_len > std::numeric_limits<qsizetype>::max()) + return false; + + uchar *data = map_sys(file, 0, data_len); + bool fromMM = !!data; + + if (!fromMM) { bool ok = false; - if (file.open(QIODevice::ReadOnly)) { - qint64 fsize = file.size(); - if (fsize <= std::numeric_limits<qsizetype>::max()) { - data_len = file.size(); - data = new uchar[data_len]; - ok = (data_len == file.read((char*)data, data_len)); - } - } + data = new uchar[data_len]; + ok = (data_len == file.read(reinterpret_cast<char *>(data), data_len)); if (!ok) { - delete [] data; + delete[] data; data = nullptr; data_len = 0; return false; } - fromMM = false; } if (data && QDynamicBufferResourceRoot::registerSelf(data, data_len)) { if (fromMM) { @@ -1221,17 +1231,17 @@ bool QDynamicFileResourceRoot::registerSelf(const QString &f) return false; } -static QString qt_resource_fixResourceRoot(QString r) { - if(!r.isEmpty()) { - if(r.startsWith(QLatin1Char(':'))) +static QString qt_resource_fixResourceRoot(QString r) +{ + if (!r.isEmpty()) { + if (r.startsWith(u':')) r = r.mid(1); - if(!r.isEmpty()) + if (!r.isEmpty()) r = QDir::cleanPath(r); } return r; } - /*! \fn bool QResource::registerResource(const QString &rccFileName, const QString &mapRoot) @@ -1242,18 +1252,18 @@ static QString qt_resource_fixResourceRoot(QString r) { \sa unregisterResource() */ -bool -QResource::registerResource(const QString &rccFilename, const QString &resourceRoot) +bool QResource::registerResource(const QString &rccFilename, const QString &resourceRoot) { QString r = qt_resource_fixResourceRoot(resourceRoot); - if(!r.isEmpty() && r[0] != QLatin1Char('/')) { - qWarning("QDir::registerResource: Registering a resource [%ls] must be rooted in an absolute path (start with /) [%ls]", + if (!r.isEmpty() && r[0] != u'/') { + qWarning("QDir::registerResource: Registering a resource [%ls] must be rooted in an " + "absolute path (start with /) [%ls]", qUtf16Printable(rccFilename), qUtf16Printable(resourceRoot)); return false; } QDynamicFileResourceRoot *root = new QDynamicFileResourceRoot(r); - if(root->registerSelf(rccFilename)) { + if (root->registerSelf(rccFilename)) { root->ref.ref(); const auto locker = qt_scoped_lock(resourceMutex()); resourceList()->append(root); @@ -1274,20 +1284,19 @@ QResource::registerResource(const QString &rccFilename, const QString &resourceR \sa registerResource() */ -bool -QResource::unregisterResource(const QString &rccFilename, const QString &resourceRoot) +bool QResource::unregisterResource(const QString &rccFilename, const QString &resourceRoot) { QString r = qt_resource_fixResourceRoot(resourceRoot); const auto locker = qt_scoped_lock(resourceMutex()); ResourceList *list = resourceList(); - for(int i = 0; i < list->size(); ++i) { + for (int i = 0; i < list->size(); ++i) { QResourceRoot *res = list->at(i); - if(res->type() == QResourceRoot::Resource_File) { - QDynamicFileResourceRoot *root = reinterpret_cast<QDynamicFileResourceRoot*>(res); + if (res->type() == QResourceRoot::Resource_File) { + QDynamicFileResourceRoot *root = reinterpret_cast<QDynamicFileResourceRoot *>(res); if (root->mappingFile() == rccFilename && root->mappingRoot() == r) { list->removeAt(i); - if(!root->ref.deref()) { + if (!root->ref.deref()) { delete root; return true; } @@ -1298,7 +1307,6 @@ QResource::unregisterResource(const QString &rccFilename, const QString &resourc return false; } - /*! \fn bool QResource::registerResource(const uchar *rccData, const QString &mapRoot) \since 4.3 @@ -1313,12 +1321,12 @@ QResource::unregisterResource(const QString &rccFilename, const QString &resourc \sa unregisterResource() */ -bool -QResource::registerResource(const uchar *rccData, const QString &resourceRoot) +bool QResource::registerResource(const uchar *rccData, const QString &resourceRoot) { QString r = qt_resource_fixResourceRoot(resourceRoot); - if(!r.isEmpty() && r[0] != QLatin1Char('/')) { - qWarning("QDir::registerResource: Registering a resource [%p] must be rooted in an absolute path (start with /) [%ls]", + if (!r.isEmpty() && r[0] != u'/') { + qWarning("QDir::registerResource: Registering a resource [%p] must be rooted in an " + "absolute path (start with /) [%ls]", rccData, qUtf16Printable(resourceRoot)); return false; } @@ -1345,20 +1353,19 @@ QResource::registerResource(const uchar *rccData, const QString &resourceRoot) \sa registerResource() */ -bool -QResource::unregisterResource(const uchar *rccData, const QString &resourceRoot) +bool QResource::unregisterResource(const uchar *rccData, const QString &resourceRoot) { QString r = qt_resource_fixResourceRoot(resourceRoot); const auto locker = qt_scoped_lock(resourceMutex()); ResourceList *list = resourceList(); - for(int i = 0; i < list->size(); ++i) { + for (int i = 0; i < list->size(); ++i) { QResourceRoot *res = list->at(i); - if(res->type() == QResourceRoot::Resource_Buffer) { - QDynamicBufferResourceRoot *root = reinterpret_cast<QDynamicBufferResourceRoot*>(res); + if (res->type() == QResourceRoot::Resource_Buffer) { + QDynamicBufferResourceRoot *root = reinterpret_cast<QDynamicBufferResourceRoot *>(res); if (root->mappingBuffer() == rccData && root->mappingRoot() == r) { list->removeAt(i); - if(!root->ref.deref()) { + if (!root->ref.deref()) { delete root; return true; } @@ -1370,7 +1377,7 @@ QResource::unregisterResource(const uchar *rccData, const QString &resourceRoot) } #if !defined(QT_BOOTSTRAPPED) -//resource engine +// resource engine class QResourceFileEnginePrivate : public QAbstractFileEnginePrivate { protected: @@ -1379,33 +1386,24 @@ private: uchar *map(qint64 offset, qint64 size, QFile::MemoryMapFlags flags); bool unmap(uchar *ptr); void uncompress() const; - qint64 offset; + void mapUncompressed(); + bool mapUncompressed_sys(); + void unmapUncompressed_sys(); + qint64 offset = 0; QResource resource; mutable QByteArray uncompressed; + bool mustUnmap = false; + + // minimum size for which we'll try to re-open ourselves in mapUncompressed() + static constexpr qsizetype RemapCompressedThreshold = 16384; protected: - QResourceFileEnginePrivate() : offset(0) { } + ~QResourceFileEnginePrivate() + { + if (mustUnmap) + unmapUncompressed_sys(); + } }; -bool QResourceFileEngine::mkdir(const QString &, bool) const -{ - return false; -} - -bool QResourceFileEngine::rmdir(const QString &, bool) const -{ - return false; -} - -bool QResourceFileEngine::setSize(qint64) -{ - return false; -} - -QStringList QResourceFileEngine::entryList(QDir::Filters filters, const QStringList &filterNames) const -{ - return QAbstractFileEngine::entryList(filters, filterNames); -} - bool QResourceFileEngine::caseSensitive() const { return true; @@ -1428,8 +1426,11 @@ void QResourceFileEngine::setFileName(const QString &file) d->resource.setFileName(file); } -bool QResourceFileEngine::open(QIODevice::OpenMode flags) +bool QResourceFileEngine::open(QIODevice::OpenMode flags, + std::optional<QFile::Permissions> permissions) { + Q_UNUSED(permissions); + Q_D(QResourceFileEngine); if (d->resource.fileName().isEmpty()) { qWarning("QResourceFileEngine::open: Missing file name"); @@ -1466,43 +1467,18 @@ bool QResourceFileEngine::flush() qint64 QResourceFileEngine::read(char *data, qint64 len) { Q_D(QResourceFileEngine); - if(len > size()-d->offset) - len = size()-d->offset; - if(len <= 0) + if (len > size() - d->offset) + len = size() - d->offset; + if (len <= 0) return 0; if (!d->uncompressed.isNull()) - memcpy(data, d->uncompressed.constData()+d->offset, len); + memcpy(data, d->uncompressed.constData() + d->offset, len); else - memcpy(data, d->resource.data()+d->offset, len); + memcpy(data, d->resource.data() + d->offset, len); d->offset += len; return len; } -qint64 QResourceFileEngine::write(const char *, qint64) -{ - return -1; -} - -bool QResourceFileEngine::remove() -{ - return false; -} - -bool QResourceFileEngine::copy(const QString &) -{ - return false; -} - -bool QResourceFileEngine::rename(const QString &) -{ - return false; -} - -bool QResourceFileEngine::link(const QString &) -{ - return false; -} - qint64 QResourceFileEngine::size() const { Q_D(const QResourceFileEngine); @@ -1518,7 +1494,7 @@ qint64 QResourceFileEngine::pos() const bool QResourceFileEngine::atEnd() const { Q_D(const QResourceFileEngine); - if(!d->resource.isValid()) + if (!d->resource.isValid()) return true; return d->offset == size(); } @@ -1526,69 +1502,61 @@ bool QResourceFileEngine::atEnd() const bool QResourceFileEngine::seek(qint64 pos) { Q_D(QResourceFileEngine); - if(!d->resource.isValid()) + if (!d->resource.isValid()) return false; - if(d->offset > size()) + if (d->offset > size()) return false; d->offset = pos; return true; } -bool QResourceFileEngine::isSequential() const -{ - return false; -} - QAbstractFileEngine::FileFlags QResourceFileEngine::fileFlags(QAbstractFileEngine::FileFlags type) const { Q_D(const QResourceFileEngine); QAbstractFileEngine::FileFlags ret; - if(!d->resource.isValid()) + if (!d->resource.isValid()) return ret; - if(type & PermsMask) - ret |= QAbstractFileEngine::FileFlags(ReadOwnerPerm|ReadUserPerm|ReadGroupPerm|ReadOtherPerm); - if(type & TypesMask) { - if(d->resource.isDir()) + if (type & PermsMask) + ret |= QAbstractFileEngine::FileFlags(ReadOwnerPerm | ReadUserPerm | ReadGroupPerm + | ReadOtherPerm); + if (type & TypesMask) { + if (d->resource.isDir()) ret |= DirectoryType; else ret |= FileType; } - if(type & FlagsMask) { + if (type & FlagsMask) { ret |= ExistsFlag; - if(d->resource.absoluteFilePath() == QLatin1String(":/")) + if (d->resource.absoluteFilePath() == ":/"_L1) ret |= RootFlag; } return ret; } -bool QResourceFileEngine::setPermissions(uint) -{ - return false; -} - QString QResourceFileEngine::fileName(FileName file) const { Q_D(const QResourceFileEngine); - if(file == BaseName) { - int slash = d->resource.fileName().lastIndexOf(QLatin1Char('/')); + if (file == BaseName) { + const qsizetype slash = d->resource.fileName().lastIndexOf(u'/'); if (slash == -1) return d->resource.fileName(); return d->resource.fileName().mid(slash + 1); - } else if(file == PathName || file == AbsolutePathName) { - const QString path = (file == AbsolutePathName) ? d->resource.absoluteFilePath() : d->resource.fileName(); - const int slash = path.lastIndexOf(QLatin1Char('/')); + } else if (file == PathName || file == AbsolutePathName) { + const QString path = (file == AbsolutePathName) ? d->resource.absoluteFilePath() + : d->resource.fileName(); + const qsizetype slash = path.lastIndexOf(u'/'); if (slash == -1) - return QLatin1String(":"); + return ":"_L1; else if (slash <= 1) - return QLatin1String(":/"); + return ":/"_L1; return path.left(slash); - } else if(file == CanonicalName || file == CanonicalPathName) { + } else if (file == CanonicalName || file == CanonicalPathName) { const QString absoluteFilePath = d->resource.absoluteFilePath(); - if(file == CanonicalPathName) { - const int slash = absoluteFilePath.lastIndexOf(QLatin1Char('/')); + if (file == CanonicalPathName) { + const qsizetype slash = absoluteFilePath.lastIndexOf(u'/'); if (slash != -1) return absoluteFilePath.left(slash); } @@ -1597,26 +1565,16 @@ QString QResourceFileEngine::fileName(FileName file) const return d->resource.fileName(); } -bool QResourceFileEngine::isRelativePath() const -{ - return false; -} - uint QResourceFileEngine::ownerId(FileOwner) const { - static const uint nobodyID = (uint) -2; + static const uint nobodyID = static_cast<uint>(-2); return nobodyID; } -QString QResourceFileEngine::owner(FileOwner) const -{ - return QString(); -} - -QDateTime QResourceFileEngine::fileTime(FileTime time) const +QDateTime QResourceFileEngine::fileTime(QFile::FileTime time) const { Q_D(const QResourceFileEngine); - if (time == ModificationTime) + if (time == QFile::FileModificationTime) return d->resource.lastModified(); return QDateTime(); } @@ -1624,31 +1582,24 @@ QDateTime QResourceFileEngine::fileTime(FileTime time) const /*! \internal */ -QAbstractFileEngine::Iterator *QResourceFileEngine::beginEntryList(QDir::Filters filters, - const QStringList &filterNames) +QAbstractFileEngine::IteratorUniquePtr +QResourceFileEngine::beginEntryList(const QString &path, QDir::Filters filters, + const QStringList &filterNames) { - return new QResourceFileEngineIterator(filters, filterNames); -} - -/*! - \internal -*/ -QAbstractFileEngine::Iterator *QResourceFileEngine::endEntryList() -{ - return nullptr; + return std::make_unique<QResourceFileEngineIterator>(path, filters, filterNames); } bool QResourceFileEngine::extension(Extension extension, const ExtensionOption *option, ExtensionReturn *output) { Q_D(QResourceFileEngine); if (extension == MapExtension) { - const MapExtensionOption *options = (const MapExtensionOption*)(option); - MapExtensionReturn *returnValue = static_cast<MapExtensionReturn*>(output); + const auto *options = static_cast<const MapExtensionOption *>(option); + auto *returnValue = static_cast<MapExtensionReturn *>(output); returnValue->address = d->map(options->offset, options->size, options->flags); return (returnValue->address != nullptr); } if (extension == UnMapExtension) { - const UnMapExtensionOption *options = (const UnMapExtensionOption*)option; + const auto *options = static_cast<const UnMapExtensionOption *>(option); return d->unmap(options->address); } return false; @@ -1662,21 +1613,27 @@ bool QResourceFileEngine::supportsExtension(Extension extension) const uchar *QResourceFileEnginePrivate::map(qint64 offset, qint64 size, QFile::MemoryMapFlags flags) { Q_Q(QResourceFileEngine); - Q_UNUSED(flags); + Q_ASSERT_X(resource.compressionAlgorithm() == QResource::NoCompression + || !uncompressed.isNull(), "QFile::map()", + "open() should have uncompressed compressed resources"); qint64 max = resource.uncompressedSize(); qint64 end; if (offset < 0 || size <= 0 || !resource.isValid() || - add_overflow(offset, size, &end) || end > max) { + qAddOverflow(offset, size, &end) || end > max) { q->setError(QFile::UnspecifiedError, QString()); return nullptr; } - const uchar *address = resource.data(); - if (resource.compressionAlgorithm() != QResource::NoCompression) { - uncompress(); - if (uncompressed.isNull()) - return nullptr; + const uchar *address = reinterpret_cast<const uchar *>(uncompressed.constBegin()); + if (!uncompressed.isNull()) + return const_cast<uchar *>(address) + offset; + + // resource was not compressed + address = resource.data(); + if (flags & QFile::MapPrivateOption) { + // We need to provide read-write memory + mapUncompressed(); address = reinterpret_cast<const uchar *>(uncompressed.constData()); } @@ -1697,6 +1654,131 @@ void QResourceFileEnginePrivate::uncompress() const uncompressed = resource.uncompressedData(); } +void QResourceFileEnginePrivate::mapUncompressed() +{ + Q_ASSERT(resource.compressionAlgorithm() == QResource::NoCompression); + if (!uncompressed.isNull()) + return; // nothing to do + + if (resource.uncompressedSize() >= RemapCompressedThreshold) { + if (mapUncompressed_sys()) + return; + } + + uncompressed = resource.uncompressedData(); + uncompressed.detach(); +} + +#if defined(MREMAP_MAYMOVE) && defined(MREMAP_DONTUNMAP) +inline bool QResourcePrivate::mayRemapData(const QResource &resource) +{ + auto d = resource.d_func(); + + // assumptions from load(): + // - d->related is not empty + // - the first item in d->related is the one with our data + // by current construction, it's also the only item + const QResourceRoot *root = d->related.at(0); + + switch (root->type()) { + case QResourceRoot::Resource_Builtin: + return true; // always acceptable, memory is read-only + case QResourceRoot::Resource_Buffer: + return false; // never acceptable, memory is heap + case QResourceRoot::Resource_File: + break; + } + + auto df = static_cast<const QDynamicFileResourceRoot *>(root); + return df->wasMemoryMapped(); +} +#endif + +// Returns the page boundaries of where \a location is located in memory. +static auto mappingBoundaries(const void *location, qsizetype size) +{ +#ifdef Q_OS_WIN + auto getpagesize = [] { + SYSTEM_INFO sysinfo; + ::GetSystemInfo(&sysinfo); + return sysinfo.dwAllocationGranularity; + }; +#endif + struct R { + void *begin; + qsizetype size; + qptrdiff offset; + } r; + + const quintptr pageMask = getpagesize() - 1; + quintptr data = quintptr(location); + quintptr begin = data & ~pageMask; + quintptr end = (data + size + pageMask) & ~pageMask; + r.begin = reinterpret_cast<void *>(begin); + r.size = end - begin; + r.offset = data & pageMask; + return r; +} + +bool QResourceFileEnginePrivate::mapUncompressed_sys() +{ + auto r = mappingBoundaries(resource.data(), resource.uncompressedSize()); + void *ptr = nullptr; + +#if defined(MREMAP_MAYMOVE) && defined(MREMAP_DONTUNMAP) + // Use MREMAP_MAYMOVE to tell the kernel to give us a new address and use + // MREMAP_DONTUNMAP (supported since kernel 5.7) to request that it create + // a new mapping of the same pages, instead of moving. We can only do that + // for pages that are read-only, otherwise the kernel replaces the source + // with pages full of nulls. + if (!QResourcePrivate::mayRemapData(resource)) + return false; + + ptr = mremap(r.begin, r.size, r.size, MREMAP_MAYMOVE | MREMAP_DONTUNMAP); + if (ptr == MAP_FAILED) + return false; + + // Allow writing, which the documentation says we allow. This is safe + // because MREMAP_DONTUNMAP only works for private mappings. + if (mprotect(ptr, r.size, PROT_READ | PROT_WRITE) != 0) { + munmap(ptr, r.size); + return false; + } +#elif defined(Q_OS_DARWIN) + mach_port_t self = mach_task_self(); + vm_address_t addr = 0; + vm_address_t mask = 0; + bool anywhere = true; + bool copy = true; + vm_prot_t cur_prot = VM_PROT_READ | VM_PROT_WRITE; + vm_prot_t max_prot = VM_PROT_ALL; + kern_return_t res = vm_remap(self, &addr, r.size, mask, anywhere, + self, vm_address_t(r.begin), copy, &cur_prot, + &max_prot, VM_INHERIT_DEFAULT); + if (res != KERN_SUCCESS) + return false; + + ptr = reinterpret_cast<void *>(addr); + if ((max_prot & VM_PROT_WRITE) == 0 || mprotect(ptr, r.size, PROT_READ | PROT_WRITE) != 0) { + munmap(ptr, r.size); + return false; + } +#endif + + if (!ptr) + return false; + const char *newdata = static_cast<char *>(ptr) + r.offset; + uncompressed = QByteArray::fromRawData(newdata, resource.uncompressedSize()); + mustUnmap = true; + return true; +} + +void QResourceFileEnginePrivate::unmapUncompressed_sys() +{ + auto r = mappingBoundaries(uncompressed.constBegin(), uncompressed.size()); + QDynamicFileResourceRoot::unmap_sys(r.begin, r.size); +} + #endif // !defined(QT_BOOTSTRAPPED) QT_END_NAMESPACE |