diff options
Diffstat (limited to 'src/network/access/qdecompresshelper.cpp')
-rw-r--r-- | src/network/access/qdecompresshelper.cpp | 238 |
1 files changed, 143 insertions, 95 deletions
diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp index d0e75ef0dc..52a0d9fc06 100644 --- a/src/network/access/qdecompresshelper.cpp +++ b/src/network/access/qdecompresshelper.cpp @@ -1,47 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtNetwork module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qdecompresshelper_p.h" -#include <QtCore/private/qbytearray_p.h> #include <QtCore/qiodevice.h> +#include <QtCore/qcoreapplication.h> +#include <limits> #include <zlib.h> #if QT_CONFIG(brotli) @@ -58,7 +23,7 @@ QT_BEGIN_NAMESPACE namespace { struct ContentEncodingMapping { - char name[8]; + QByteArrayView name; QDecompressHelper::ContentEncoding encoding; }; @@ -73,10 +38,10 @@ constexpr ContentEncodingMapping contentEncodingMapping[] { { "deflate", QDecompressHelper::Deflate }, }; -QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept +QDecompressHelper::ContentEncoding encodingFromByteArray(QByteArrayView ce) noexcept { for (const auto &mapping : contentEncodingMapping) { - if (ce.compare(QByteArrayView(mapping.name, strlen(mapping.name)), Qt::CaseInsensitive) == 0) + if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0) return mapping.encoding; } return QDecompressHelper::None; @@ -102,22 +67,19 @@ ZSTD_DStream *toZstandardPointer(void *ptr) #endif } -bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding) +bool QDecompressHelper::isSupportedEncoding(QByteArrayView encoding) { return encodingFromByteArray(encoding) != QDecompressHelper::None; } QByteArrayList QDecompressHelper::acceptedEncoding() { - static QByteArrayList accepted = []() { - QByteArrayList list; - list.reserve(sizeof(contentEncodingMapping) / sizeof(contentEncodingMapping[0])); - for (const auto &mapping : contentEncodingMapping) { - list << QByteArray(mapping.name); - } - return list; - }(); - return accepted; + QByteArrayList list; + list.reserve(std::size(contentEncodingMapping)); + for (const auto &mapping : contentEncodingMapping) { + list << mapping.name.toByteArray(); + } + return list; } QDecompressHelper::~QDecompressHelper() @@ -125,18 +87,21 @@ QDecompressHelper::~QDecompressHelper() clear(); } -bool QDecompressHelper::setEncoding(const QByteArray &encoding) +bool QDecompressHelper::setEncoding(QByteArrayView encoding) { Q_ASSERT(contentEncoding == QDecompressHelper::None); if (contentEncoding != QDecompressHelper::None) { qWarning("Encoding is already set."); + // This isn't an error, so it doesn't set errorStr, it's just wrong usage. return false; } ContentEncoding ce = encodingFromByteArray(encoding); if (ce == None) { - qWarning("An unsupported content encoding was selected: %s", encoding.data()); + errorStr = QCoreApplication::translate("QHttp", "Unsupported content encoding: %1") + .arg(QLatin1String(encoding)); return false; } + errorStr = QString(); // clear error return setEncoding(ce); } @@ -178,7 +143,8 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce) break; } if (!decoderPointer) { - qWarning("Failed to initialize the decoder."); + errorStr = QCoreApplication::translate("QHttp", + "Failed to initialize the compression decoder."); contentEncoding = QDecompressHelper::None; return false; } @@ -236,7 +202,13 @@ void QDecompressHelper::setCountingBytesEnabled(bool shouldCount) qint64 QDecompressHelper::uncompressedSize() const { Q_ASSERT(countDecompressed); - return uncompressedBytes; + // Use the 'totalUncompressedBytes' from the countHelper if it exceeds the amount of bytes + // that we know about. + auto totalUncompressed = + countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes + ? countHelper->totalUncompressedBytes + : totalUncompressedBytes; + return totalUncompressed - totalBytesRead; } /*! @@ -261,10 +233,9 @@ void QDecompressHelper::feed(QByteArray &&data) { Q_ASSERT(contentEncoding != None); totalCompressedBytes += data.size(); - if (!countInternal(data)) + compressedDataBuffer.append(std::move(data)); + if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1])) clear(); // If our counting brother failed then so will we :| - else - compressedDataBuffer.append(std::move(data)); } /*! @@ -275,10 +246,9 @@ void QDecompressHelper::feed(const QByteDataBuffer &buffer) { Q_ASSERT(contentEncoding != None); totalCompressedBytes += buffer.byteAmount(); + compressedDataBuffer.append(buffer); if (!countInternal(buffer)) clear(); // If our counting brother failed then so will we :| - else - compressedDataBuffer.append(buffer); } /*! @@ -289,10 +259,10 @@ void QDecompressHelper::feed(QByteDataBuffer &&buffer) { Q_ASSERT(contentEncoding != None); totalCompressedBytes += buffer.byteAmount(); - if (!countInternal(buffer)) + const QByteDataBuffer copy(buffer); + compressedDataBuffer.append(std::move(buffer)); + if (!countInternal(copy)) clear(); // If our counting brother failed then so will we :| - else - compressedDataBuffer.append(std::move(buffer)); } /*! @@ -302,19 +272,34 @@ void QDecompressHelper::feed(QByteDataBuffer &&buffer) This lets us know the final size, unfortunately at the cost of increased computation. - Potential @future improvement: - Decompress XX MiB/KiB before starting the count. - For smaller files the extra decompression can then be avoided. + To save on some of the computation we will store the data until + we reach \c MaxDecompressedDataBufferSize stored. In this case the + "penalty" is completely removed from users who read the data on + readyRead rather than waiting for it all to be received. And + any file smaller than \c MaxDecompressedDataBufferSize will + avoid this issue as well. */ bool QDecompressHelper::countInternal() { Q_ASSERT(countDecompressed); + while (hasDataInternal() + && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) { + const qsizetype toRead = 256 * 1024; + QByteArray buffer(toRead, Qt::Uninitialized); + qsizetype bytesRead = readInternal(buffer.data(), buffer.size()); + if (bytesRead == -1) + return false; + buffer.truncate(bytesRead); + decompressedDataBuffer.append(std::move(buffer)); + } + if (!hasDataInternal()) + return true; // handled all the data so far, just return + while (countHelper->hasData()) { std::array<char, 1024> temp; qsizetype bytesRead = countHelper->read(temp.data(), temp.size()); if (bytesRead == -1) return false; - uncompressedBytes += bytesRead; } return true; } @@ -328,7 +313,7 @@ bool QDecompressHelper::countInternal(const QByteArray &data) if (countDecompressed) { if (!countHelper) { countHelper = std::make_unique<QDecompressHelper>(); - countHelper->setArchiveBombDetectionEnabled(archiveBombDetectionEnabled); + countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold); countHelper->setEncoding(contentEncoding); } countHelper->feed(data); @@ -346,7 +331,7 @@ bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer) if (countDecompressed) { if (!countHelper) { countHelper = std::make_unique<QDecompressHelper>(); - countHelper->setArchiveBombDetectionEnabled(archiveBombDetectionEnabled); + countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold); countHelper->setEncoding(contentEncoding); } countHelper->feed(buffer); @@ -357,13 +342,45 @@ bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer) qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) { + if (maxSize <= 0) + return 0; + if (!isValid()) return -1; - qsizetype bytesRead = -1; if (!hasData()) return 0; + qsizetype cachedRead = 0; + if (!decompressedDataBuffer.isEmpty()) { + cachedRead = decompressedDataBuffer.read(data, maxSize); + data += cachedRead; + maxSize -= cachedRead; + } + + qsizetype bytesRead = readInternal(data, maxSize); + if (bytesRead == -1) + return -1; + totalBytesRead += bytesRead + cachedRead; + return bytesRead + cachedRead; +} + +/*! + \internal + Like read() but without attempting to read the + cached/already-decompressed data. +*/ +qsizetype QDecompressHelper::readInternal(char *data, qsizetype maxSize) +{ + Q_ASSERT(isValid()); + + if (maxSize <= 0) + return 0; + + if (!hasDataInternal()) + return 0; + + qsizetype bytesRead = -1; switch (contentEncoding) { case None: Q_UNREACHABLE(); @@ -381,44 +398,38 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) } if (bytesRead == -1) clear(); - else if (countDecompressed) - uncompressedBytes -= bytesRead; totalUncompressedBytes += bytesRead; - if (isPotentialArchiveBomb()) + if (isPotentialArchiveBomb()) { + errorStr = QCoreApplication::translate( + "QHttp", + "The decompressed output exceeds the limits specified by " + "QNetworkRequest::decompressedSafetyCheckThreshold()"); return -1; + } return bytesRead; } /*! \internal - Disables or enables checking the decompression ratio of archives - according to the value of \a enable. - Only for enabling us to test handling of large decompressed files - without needing to bundle large compressed files. + Set the \a threshold required before the archive bomb detection kicks in. + By default this is 10MB. Setting it to -1 is treated as disabling the + feature. */ -void QDecompressHelper::setArchiveBombDetectionEnabled(bool enable) -{ - archiveBombDetectionEnabled = enable; - if (countHelper) - countHelper->setArchiveBombDetectionEnabled(enable); -} - -void QDecompressHelper::setMinimumArchiveBombSize(qint64 threshold) +void QDecompressHelper::setDecompressedSafetyCheckThreshold(qint64 threshold) { - minimumArchiveBombSize = threshold; + if (threshold == -1) + threshold = std::numeric_limits<qint64>::max(); + archiveBombCheckThreshold = threshold; } bool QDecompressHelper::isPotentialArchiveBomb() const { - if (!archiveBombDetectionEnabled) - return false; - if (totalCompressedBytes == 0) return false; - if (totalUncompressedBytes <= minimumArchiveBombSize) + if (totalUncompressedBytes <= archiveBombCheckThreshold) return false; // Some protection against malicious or corrupted compressed files that expand far more than @@ -430,12 +441,16 @@ bool QDecompressHelper::isPotentialArchiveBomb() const break; case Deflate: case GZip: + // This value is mentioned in docs for + // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized if (ratio > 40) { return true; } break; case Brotli: case Zstandard: + // This value is mentioned in docs for + // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized if (ratio > 100) { return true; } @@ -454,6 +469,16 @@ bool QDecompressHelper::isPotentialArchiveBomb() const */ bool QDecompressHelper::hasData() const { + return hasDataInternal() || !decompressedDataBuffer.isEmpty(); +} + +/*! + \internal + Like hasData() but internally the buffer of decompressed data is + not interesting. +*/ +bool QDecompressHelper::hasDataInternal() const +{ return encodedBytesAvailable() || decoderHasData; } @@ -462,11 +487,29 @@ qint64 QDecompressHelper::encodedBytesAvailable() const return compressedDataBuffer.byteAmount(); } +/*! + \internal + Returns whether or not the object is valid. + If it becomes invalid after an operation has been performed + then an error has occurred. + \sa errorString() +*/ bool QDecompressHelper::isValid() const { return contentEncoding != None; } +/*! + \internal + Returns a string describing the error that occurred or an empty + string if no error occurred. + \sa isValid() +*/ +QString QDecompressHelper::errorString() const +{ + return errorStr; +} + void QDecompressHelper::clear() { switch (contentEncoding) { @@ -501,13 +544,16 @@ void QDecompressHelper::clear() contentEncoding = None; compressedDataBuffer.clear(); + decompressedDataBuffer.clear(); decoderHasData = false; countDecompressed = false; countHelper.reset(); - uncompressedBytes = 0; + totalBytesRead = 0; totalUncompressedBytes = 0; totalCompressedBytes = 0; + + errorStr.clear(); } qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) @@ -663,8 +709,9 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) switch (result) { case BROTLI_DECODER_RESULT_ERROR: - qWarning("Brotli error: %s", - BrotliDecoderErrorString(BrotliDecoderGetErrorCode(brotliDecoderState))); + errorStr = QLatin1String("Brotli error: %1") + .arg(QString::fromUtf8(BrotliDecoderErrorString( + BrotliDecoderGetErrorCode(brotliDecoderState)))); return -1; case BROTLI_DECODER_RESULT_SUCCESS: BrotliDecoderDestroyInstance(brotliDecoderState); @@ -710,7 +757,8 @@ qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize) while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) { size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf); if (ZSTD_isError(retValue)) { - qWarning("ZStandard error: %s", ZSTD_getErrorName(retValue)); + errorStr = QLatin1String("ZStandard error: %1") + .arg(QString::fromUtf8(ZSTD_getErrorName(retValue))); return -1; } else { decoderHasData = false; |