diff options
Diffstat (limited to 'src/network/access/qdecompresshelper.cpp')
-rw-r--r-- | src/network/access/qdecompresshelper.cpp | 335 |
1 files changed, 217 insertions, 118 deletions
diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp index 66ac70fec5..52a0d9fc06 100644 --- a/src/network/access/qdecompresshelper.cpp +++ b/src/network/access/qdecompresshelper.cpp @@ -1,47 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtNetwork module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qdecompresshelper_p.h" -#include <QtCore/private/qbytearray_p.h> #include <QtCore/qiodevice.h> +#include <QtCore/qcoreapplication.h> +#include <limits> #include <zlib.h> #if QT_CONFIG(brotli) @@ -58,7 +23,7 @@ QT_BEGIN_NAMESPACE namespace { struct ContentEncodingMapping { - char name[8]; + QByteArrayView name; QDecompressHelper::ContentEncoding encoding; }; @@ -73,10 +38,10 @@ constexpr ContentEncodingMapping contentEncodingMapping[] { { "deflate", QDecompressHelper::Deflate }, }; -QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept +QDecompressHelper::ContentEncoding encodingFromByteArray(QByteArrayView ce) noexcept { for (const auto &mapping : contentEncodingMapping) { - if (ce.compare(QByteArrayView(mapping.name, strlen(mapping.name)), Qt::CaseInsensitive) == 0) + if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0) return mapping.encoding; } return QDecompressHelper::None; @@ -102,22 +67,19 @@ ZSTD_DStream *toZstandardPointer(void *ptr) #endif } -bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding) +bool QDecompressHelper::isSupportedEncoding(QByteArrayView encoding) { return encodingFromByteArray(encoding) != QDecompressHelper::None; } QByteArrayList QDecompressHelper::acceptedEncoding() { - static QByteArrayList accepted = []() { - QByteArrayList list; - list.reserve(sizeof(contentEncodingMapping) / sizeof(contentEncodingMapping[0])); - for (const auto &mapping : contentEncodingMapping) { - list << QByteArray(mapping.name); - } - return list; - }(); - return accepted; + QByteArrayList list; + list.reserve(std::size(contentEncodingMapping)); + for (const auto &mapping : contentEncodingMapping) { + list << mapping.name.toByteArray(); + } + return list; } QDecompressHelper::~QDecompressHelper() @@ -125,18 +87,21 @@ QDecompressHelper::~QDecompressHelper() clear(); } -bool QDecompressHelper::setEncoding(const QByteArray &encoding) +bool QDecompressHelper::setEncoding(QByteArrayView encoding) { Q_ASSERT(contentEncoding == QDecompressHelper::None); if (contentEncoding != QDecompressHelper::None) { qWarning("Encoding is already set."); + // This isn't an error, so it doesn't set errorStr, it's just wrong usage. return false; } ContentEncoding ce = encodingFromByteArray(encoding); if (ce == None) { - qWarning("An unsupported content encoding was selected: %s", encoding.data()); + errorStr = QCoreApplication::translate("QHttp", "Unsupported content encoding: %1") + .arg(QLatin1String(encoding)); return false; } + errorStr = QString(); // clear error return setEncoding(ce); } @@ -178,7 +143,8 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce) break; } if (!decoderPointer) { - qWarning("Failed to initialize the decoder."); + errorStr = QCoreApplication::translate("QHttp", + "Failed to initialize the compression decoder."); contentEncoding = QDecompressHelper::None; return false; } @@ -236,7 +202,13 @@ void QDecompressHelper::setCountingBytesEnabled(bool shouldCount) qint64 QDecompressHelper::uncompressedSize() const { Q_ASSERT(countDecompressed); - return uncompressedBytes; + // Use the 'totalUncompressedBytes' from the countHelper if it exceeds the amount of bytes + // that we know about. + auto totalUncompressed = + countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes + ? countHelper->totalUncompressedBytes + : totalUncompressedBytes; + return totalUncompressed - totalBytesRead; } /*! @@ -260,10 +232,10 @@ void QDecompressHelper::feed(const QByteArray &data) void QDecompressHelper::feed(QByteArray &&data) { Q_ASSERT(contentEncoding != None); - if (!countInternal(data)) + totalCompressedBytes += data.size(); + compressedDataBuffer.append(std::move(data)); + if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1])) clear(); // If our counting brother failed then so will we :| - else - compressedDataBuffer.append(std::move(data)); } /*! @@ -273,10 +245,10 @@ void QDecompressHelper::feed(QByteArray &&data) void QDecompressHelper::feed(const QByteDataBuffer &buffer) { Q_ASSERT(contentEncoding != None); + totalCompressedBytes += buffer.byteAmount(); + compressedDataBuffer.append(buffer); if (!countInternal(buffer)) clear(); // If our counting brother failed then so will we :| - else - compressedDataBuffer.append(buffer); } /*! @@ -286,10 +258,11 @@ void QDecompressHelper::feed(const QByteDataBuffer &buffer) void QDecompressHelper::feed(QByteDataBuffer &&buffer) { Q_ASSERT(contentEncoding != None); - if (!countInternal(buffer)) + totalCompressedBytes += buffer.byteAmount(); + const QByteDataBuffer copy(buffer); + compressedDataBuffer.append(std::move(buffer)); + if (!countInternal(copy)) clear(); // If our counting brother failed then so will we :| - else - compressedDataBuffer.append(std::move(buffer)); } /*! @@ -299,19 +272,34 @@ void QDecompressHelper::feed(QByteDataBuffer &&buffer) This lets us know the final size, unfortunately at the cost of increased computation. - Potential @future improvement: - Decompress XX MiB/KiB before starting the count. - For smaller files the extra decompression can then be avoided. + To save on some of the computation we will store the data until + we reach \c MaxDecompressedDataBufferSize stored. In this case the + "penalty" is completely removed from users who read the data on + readyRead rather than waiting for it all to be received. And + any file smaller than \c MaxDecompressedDataBufferSize will + avoid this issue as well. */ bool QDecompressHelper::countInternal() { Q_ASSERT(countDecompressed); + while (hasDataInternal() + && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) { + const qsizetype toRead = 256 * 1024; + QByteArray buffer(toRead, Qt::Uninitialized); + qsizetype bytesRead = readInternal(buffer.data(), buffer.size()); + if (bytesRead == -1) + return false; + buffer.truncate(bytesRead); + decompressedDataBuffer.append(std::move(buffer)); + } + if (!hasDataInternal()) + return true; // handled all the data so far, just return + while (countHelper->hasData()) { std::array<char, 1024> temp; qsizetype bytesRead = countHelper->read(temp.data(), temp.size()); if (bytesRead == -1) return false; - uncompressedBytes += bytesRead; } return true; } @@ -325,6 +313,7 @@ bool QDecompressHelper::countInternal(const QByteArray &data) if (countDecompressed) { if (!countHelper) { countHelper = std::make_unique<QDecompressHelper>(); + countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold); countHelper->setEncoding(contentEncoding); } countHelper->feed(data); @@ -342,6 +331,7 @@ bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer) if (countDecompressed) { if (!countHelper) { countHelper = std::make_unique<QDecompressHelper>(); + countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold); countHelper->setEncoding(contentEncoding); } countHelper->feed(buffer); @@ -352,13 +342,45 @@ bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer) qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) { + if (maxSize <= 0) + return 0; + if (!isValid()) return -1; - qsizetype bytesRead = -1; if (!hasData()) return 0; + qsizetype cachedRead = 0; + if (!decompressedDataBuffer.isEmpty()) { + cachedRead = decompressedDataBuffer.read(data, maxSize); + data += cachedRead; + maxSize -= cachedRead; + } + + qsizetype bytesRead = readInternal(data, maxSize); + if (bytesRead == -1) + return -1; + totalBytesRead += bytesRead + cachedRead; + return bytesRead + cachedRead; +} + +/*! + \internal + Like read() but without attempting to read the + cached/already-decompressed data. +*/ +qsizetype QDecompressHelper::readInternal(char *data, qsizetype maxSize) +{ + Q_ASSERT(isValid()); + + if (maxSize <= 0) + return 0; + + if (!hasDataInternal()) + return 0; + + qsizetype bytesRead = -1; switch (contentEncoding) { case None: Q_UNREACHABLE(); @@ -376,13 +398,69 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) } if (bytesRead == -1) clear(); - else if (countDecompressed) - uncompressedBytes -= bytesRead; + + totalUncompressedBytes += bytesRead; + if (isPotentialArchiveBomb()) { + errorStr = QCoreApplication::translate( + "QHttp", + "The decompressed output exceeds the limits specified by " + "QNetworkRequest::decompressedSafetyCheckThreshold()"); + return -1; + } + return bytesRead; } /*! \internal + Set the \a threshold required before the archive bomb detection kicks in. + By default this is 10MB. Setting it to -1 is treated as disabling the + feature. +*/ +void QDecompressHelper::setDecompressedSafetyCheckThreshold(qint64 threshold) +{ + if (threshold == -1) + threshold = std::numeric_limits<qint64>::max(); + archiveBombCheckThreshold = threshold; +} + +bool QDecompressHelper::isPotentialArchiveBomb() const +{ + if (totalCompressedBytes == 0) + return false; + + if (totalUncompressedBytes <= archiveBombCheckThreshold) + return false; + + // Some protection against malicious or corrupted compressed files that expand far more than + // is reasonable. + double ratio = double(totalUncompressedBytes) / double(totalCompressedBytes); + switch (contentEncoding) { + case None: + Q_UNREACHABLE(); + break; + case Deflate: + case GZip: + // This value is mentioned in docs for + // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized + if (ratio > 40) { + return true; + } + break; + case Brotli: + case Zstandard: + // This value is mentioned in docs for + // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized + if (ratio > 100) { + return true; + } + break; + } + return false; +} + +/*! + \internal Returns true if there are encoded bytes left or there is some indication that the decoder still has data left internally. @@ -391,6 +469,16 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) */ bool QDecompressHelper::hasData() const { + return hasDataInternal() || !decompressedDataBuffer.isEmpty(); +} + +/*! + \internal + Like hasData() but internally the buffer of decompressed data is + not interesting. +*/ +bool QDecompressHelper::hasDataInternal() const +{ return encodedBytesAvailable() || decoderHasData; } @@ -399,11 +487,29 @@ qint64 QDecompressHelper::encodedBytesAvailable() const return compressedDataBuffer.byteAmount(); } +/*! + \internal + Returns whether or not the object is valid. + If it becomes invalid after an operation has been performed + then an error has occurred. + \sa errorString() +*/ bool QDecompressHelper::isValid() const { return contentEncoding != None; } +/*! + \internal + Returns a string describing the error that occurred or an empty + string if no error occurred. + \sa isValid() +*/ +QString QDecompressHelper::errorString() const +{ + return errorStr; +} + void QDecompressHelper::clear() { switch (contentEncoding) { @@ -438,11 +544,16 @@ void QDecompressHelper::clear() contentEncoding = None; compressedDataBuffer.clear(); + decompressedDataBuffer.clear(); decoderHasData = false; countDecompressed = false; countHelper.reset(); - uncompressedBytes = 0; + totalBytesRead = 0; + totalUncompressedBytes = 0; + totalCompressedBytes = 0; + + errorStr.clear(); } qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) @@ -453,16 +564,12 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) static const size_t zlibMaxSize = size_t(std::numeric_limits<decltype(inflateStream->avail_in)>::max()); - QByteArray input; - if (!compressedDataBuffer.isEmpty()) { - if (zlibMaxSize < size_t(compressedDataBuffer.sizeNextBlock())) - input = compressedDataBuffer.read(zlibMaxSize); - else - input = compressedDataBuffer.read(); - } + QByteArrayView input = compressedDataBuffer.readPointer(); + if (size_t(input.size()) > zlibMaxSize) + input = input.sliced(zlibMaxSize); inflateStream->avail_in = input.size(); - inflateStream->next_in = reinterpret_cast<Bytef *>(input.data()); + inflateStream->next_in = reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); bool bigMaxSize = (zlibMaxSize < size_t(maxSize)); qsizetype adjustedAvailableOut = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize; @@ -490,7 +597,8 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) return -1; } else { inflateStream->avail_in = input.size(); - inflateStream->next_in = reinterpret_cast<Bytef *>(input.data()); + inflateStream->next_in = + reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); continue; } } else if (ret < 0 || ret == Z_NEED_DICT) { @@ -512,6 +620,7 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) delete inflateStream; decoderPointer = nullptr; // Failed to reinitialize, so we'll just return what we have + compressedDataBuffer.advanceReadPointer(input.size() - avail_in); return bytesDecoded; } else { inflateStream->next_in = next_in; @@ -520,6 +629,7 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) } } else { // No extra data, stream is at the end. We're done. + compressedDataBuffer.advanceReadPointer(input.size()); return bytesDecoded; } } @@ -532,23 +642,19 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) inflateStream->next_out = reinterpret_cast<Bytef *>(data + bytesDecoded); } - if (inflateStream->avail_in == 0 && inflateStream->avail_out > 0 - && !compressedDataBuffer.isEmpty()) { + if (inflateStream->avail_in == 0 && inflateStream->avail_out > 0) { // Grab the next input! - if (zlibMaxSize < size_t(compressedDataBuffer.sizeNextBlock())) - input = compressedDataBuffer.read(zlibMaxSize); - else - input = compressedDataBuffer.read(); + compressedDataBuffer.advanceReadPointer(input.size()); + input = compressedDataBuffer.readPointer(); + if (size_t(input.size()) > zlibMaxSize) + input = input.sliced(zlibMaxSize); inflateStream->avail_in = input.size(); - inflateStream->next_in = reinterpret_cast<Bytef *>(input.data()); + inflateStream->next_in = + reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); } } while (inflateStream->avail_out > 0 && inflateStream->avail_in > 0); - if (inflateStream->avail_in) { - // Some input was left unused; move back to the buffer - input = input.right(inflateStream->avail_in); - compressedDataBuffer.prepend(input); - } + compressedDataBuffer.advanceReadPointer(input.size() - inflateStream->avail_in); return bytesDecoded; } @@ -588,10 +694,8 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) return bytesDecoded; Q_ASSERT(bytesDecoded < maxSize); - QByteArray input; - if (!compressedDataBuffer.isEmpty()) - input = compressedDataBuffer.read(); - const uint8_t *encodedPtr = reinterpret_cast<const uint8_t *>(input.constData()); + QByteArrayView input = compressedDataBuffer.readPointer(); + const uint8_t *encodedPtr = reinterpret_cast<const uint8_t *>(input.data()); size_t encodedBytesRemaining = input.size(); uint8_t *decodedPtr = reinterpret_cast<uint8_t *>(data + bytesDecoded); @@ -605,16 +709,19 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) switch (result) { case BROTLI_DECODER_RESULT_ERROR: - qWarning("Brotli error: %s", - BrotliDecoderErrorString(BrotliDecoderGetErrorCode(brotliDecoderState))); + errorStr = QLatin1String("Brotli error: %1") + .arg(QString::fromUtf8(BrotliDecoderErrorString( + BrotliDecoderGetErrorCode(brotliDecoderState)))); return -1; case BROTLI_DECODER_RESULT_SUCCESS: BrotliDecoderDestroyInstance(brotliDecoderState); decoderPointer = nullptr; + compressedDataBuffer.clear(); return bytesDecoded; case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: - if (!compressedDataBuffer.isEmpty()) { - input = compressedDataBuffer.read(); + compressedDataBuffer.advanceReadPointer(input.size()); + input = compressedDataBuffer.readPointer(); + if (!input.isEmpty()) { encodedPtr = reinterpret_cast<const uint8_t *>(input.constData()); encodedBytesRemaining = input.size(); break; @@ -627,11 +734,7 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) break; } } - if (encodedBytesRemaining) { - // Some input was left unused; move back to the buffer - input = input.right(QByteArray::size_type(encodedBytesRemaining)); - compressedDataBuffer.prepend(input); - } + compressedDataBuffer.advanceReadPointer(input.size() - encodedBytesRemaining); return bytesDecoded; #endif } @@ -645,10 +748,8 @@ qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize) #else ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer); - QByteArray input; - if (!compressedDataBuffer.isEmpty()) - input = compressedDataBuffer.read(); - ZSTD_inBuffer inBuf { input.constData(), size_t(input.size()), 0 }; + QByteArrayView input = compressedDataBuffer.readPointer(); + ZSTD_inBuffer inBuf { input.data(), size_t(input.size()), 0 }; ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 }; @@ -656,7 +757,8 @@ qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize) while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) { size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf); if (ZSTD_isError(retValue)) { - qWarning("ZStandard error: %s", ZSTD_getErrorName(retValue)); + errorStr = QLatin1String("ZStandard error: %1") + .arg(QString::fromUtf8(ZSTD_getErrorName(retValue))); return -1; } else { decoderHasData = false; @@ -664,17 +766,14 @@ qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize) // if pos == size then there may be data left over in internal buffers if (outBuf.pos == outBuf.size) { decoderHasData = true; - } else if (inBuf.pos == inBuf.size && !compressedDataBuffer.isEmpty()) { - input = compressedDataBuffer.read(); + } else if (inBuf.pos == inBuf.size) { + compressedDataBuffer.advanceReadPointer(input.size()); + input = compressedDataBuffer.readPointer(); inBuf = { input.constData(), size_t(input.size()), 0 }; } } } - if (inBuf.pos < inBuf.size) { - // Some input was left unused; move back to the buffer - input = input.mid(QByteArray::size_type(inBuf.pos)); - compressedDataBuffer.prepend(std::move(input)); - } + compressedDataBuffer.advanceReadPointer(inBuf.pos); return bytesDecoded; #endif } |