summaryrefslogtreecommitdiffstats
path: root/src/network/access/qdecompresshelper.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/network/access/qdecompresshelper.cpp')
-rw-r--r--src/network/access/qdecompresshelper.cpp238
1 files changed, 143 insertions, 95 deletions
diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp
index d0e75ef0dc..52a0d9fc06 100644
--- a/src/network/access/qdecompresshelper.cpp
+++ b/src/network/access/qdecompresshelper.cpp
@@ -1,47 +1,12 @@
-/****************************************************************************
-**
-** Copyright (C) 2020 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the QtNetwork module of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:LGPL$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU Lesser General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU Lesser
-** General Public License version 3 as published by the Free Software
-** Foundation and appearing in the file LICENSE.LGPL3 included in the
-** packaging of this file. Please review the following information to
-** ensure the GNU Lesser General Public License version 3 requirements
-** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 2.0 or (at your option) the GNU General
-** Public license version 3 or any later version approved by the KDE Free
-** Qt Foundation. The licenses are as published by the Free Software
-** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-2.0.html and
-** https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
+// Copyright (C) 2020 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qdecompresshelper_p.h"
-#include <QtCore/private/qbytearray_p.h>
#include <QtCore/qiodevice.h>
+#include <QtCore/qcoreapplication.h>
+#include <limits>
#include <zlib.h>
#if QT_CONFIG(brotli)
@@ -58,7 +23,7 @@ QT_BEGIN_NAMESPACE
namespace {
struct ContentEncodingMapping
{
- char name[8];
+ QByteArrayView name;
QDecompressHelper::ContentEncoding encoding;
};
@@ -73,10 +38,10 @@ constexpr ContentEncodingMapping contentEncodingMapping[] {
{ "deflate", QDecompressHelper::Deflate },
};
-QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept
+QDecompressHelper::ContentEncoding encodingFromByteArray(QByteArrayView ce) noexcept
{
for (const auto &mapping : contentEncodingMapping) {
- if (ce.compare(QByteArrayView(mapping.name, strlen(mapping.name)), Qt::CaseInsensitive) == 0)
+ if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0)
return mapping.encoding;
}
return QDecompressHelper::None;
@@ -102,22 +67,19 @@ ZSTD_DStream *toZstandardPointer(void *ptr)
#endif
}
-bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding)
+bool QDecompressHelper::isSupportedEncoding(QByteArrayView encoding)
{
return encodingFromByteArray(encoding) != QDecompressHelper::None;
}
QByteArrayList QDecompressHelper::acceptedEncoding()
{
- static QByteArrayList accepted = []() {
- QByteArrayList list;
- list.reserve(sizeof(contentEncodingMapping) / sizeof(contentEncodingMapping[0]));
- for (const auto &mapping : contentEncodingMapping) {
- list << QByteArray(mapping.name);
- }
- return list;
- }();
- return accepted;
+ QByteArrayList list;
+ list.reserve(std::size(contentEncodingMapping));
+ for (const auto &mapping : contentEncodingMapping) {
+ list << mapping.name.toByteArray();
+ }
+ return list;
}
QDecompressHelper::~QDecompressHelper()
@@ -125,18 +87,21 @@ QDecompressHelper::~QDecompressHelper()
clear();
}
-bool QDecompressHelper::setEncoding(const QByteArray &encoding)
+bool QDecompressHelper::setEncoding(QByteArrayView encoding)
{
Q_ASSERT(contentEncoding == QDecompressHelper::None);
if (contentEncoding != QDecompressHelper::None) {
qWarning("Encoding is already set.");
+ // This isn't an error, so it doesn't set errorStr, it's just wrong usage.
return false;
}
ContentEncoding ce = encodingFromByteArray(encoding);
if (ce == None) {
- qWarning("An unsupported content encoding was selected: %s", encoding.data());
+ errorStr = QCoreApplication::translate("QHttp", "Unsupported content encoding: %1")
+ .arg(QLatin1String(encoding));
return false;
}
+ errorStr = QString(); // clear error
return setEncoding(ce);
}
@@ -178,7 +143,8 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce)
break;
}
if (!decoderPointer) {
- qWarning("Failed to initialize the decoder.");
+ errorStr = QCoreApplication::translate("QHttp",
+ "Failed to initialize the compression decoder.");
contentEncoding = QDecompressHelper::None;
return false;
}
@@ -236,7 +202,13 @@ void QDecompressHelper::setCountingBytesEnabled(bool shouldCount)
qint64 QDecompressHelper::uncompressedSize() const
{
Q_ASSERT(countDecompressed);
- return uncompressedBytes;
+ // Use the 'totalUncompressedBytes' from the countHelper if it exceeds the amount of bytes
+ // that we know about.
+ auto totalUncompressed =
+ countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes
+ ? countHelper->totalUncompressedBytes
+ : totalUncompressedBytes;
+ return totalUncompressed - totalBytesRead;
}
/*!
@@ -261,10 +233,9 @@ void QDecompressHelper::feed(QByteArray &&data)
{
Q_ASSERT(contentEncoding != None);
totalCompressedBytes += data.size();
- if (!countInternal(data))
+ compressedDataBuffer.append(std::move(data));
+ if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1]))
clear(); // If our counting brother failed then so will we :|
- else
- compressedDataBuffer.append(std::move(data));
}
/*!
@@ -275,10 +246,9 @@ void QDecompressHelper::feed(const QByteDataBuffer &buffer)
{
Q_ASSERT(contentEncoding != None);
totalCompressedBytes += buffer.byteAmount();
+ compressedDataBuffer.append(buffer);
if (!countInternal(buffer))
clear(); // If our counting brother failed then so will we :|
- else
- compressedDataBuffer.append(buffer);
}
/*!
@@ -289,10 +259,10 @@ void QDecompressHelper::feed(QByteDataBuffer &&buffer)
{
Q_ASSERT(contentEncoding != None);
totalCompressedBytes += buffer.byteAmount();
- if (!countInternal(buffer))
+ const QByteDataBuffer copy(buffer);
+ compressedDataBuffer.append(std::move(buffer));
+ if (!countInternal(copy))
clear(); // If our counting brother failed then so will we :|
- else
- compressedDataBuffer.append(std::move(buffer));
}
/*!
@@ -302,19 +272,34 @@ void QDecompressHelper::feed(QByteDataBuffer &&buffer)
This lets us know the final size, unfortunately at the cost of
increased computation.
- Potential @future improvement:
- Decompress XX MiB/KiB before starting the count.
- For smaller files the extra decompression can then be avoided.
+ To save on some of the computation we will store the data until
+ we reach \c MaxDecompressedDataBufferSize stored. In this case the
+ "penalty" is completely removed from users who read the data on
+ readyRead rather than waiting for it all to be received. And
+ any file smaller than \c MaxDecompressedDataBufferSize will
+ avoid this issue as well.
*/
bool QDecompressHelper::countInternal()
{
Q_ASSERT(countDecompressed);
+ while (hasDataInternal()
+ && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) {
+ const qsizetype toRead = 256 * 1024;
+ QByteArray buffer(toRead, Qt::Uninitialized);
+ qsizetype bytesRead = readInternal(buffer.data(), buffer.size());
+ if (bytesRead == -1)
+ return false;
+ buffer.truncate(bytesRead);
+ decompressedDataBuffer.append(std::move(buffer));
+ }
+ if (!hasDataInternal())
+ return true; // handled all the data so far, just return
+
while (countHelper->hasData()) {
std::array<char, 1024> temp;
qsizetype bytesRead = countHelper->read(temp.data(), temp.size());
if (bytesRead == -1)
return false;
- uncompressedBytes += bytesRead;
}
return true;
}
@@ -328,7 +313,7 @@ bool QDecompressHelper::countInternal(const QByteArray &data)
if (countDecompressed) {
if (!countHelper) {
countHelper = std::make_unique<QDecompressHelper>();
- countHelper->setArchiveBombDetectionEnabled(archiveBombDetectionEnabled);
+ countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
countHelper->setEncoding(contentEncoding);
}
countHelper->feed(data);
@@ -346,7 +331,7 @@ bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer)
if (countDecompressed) {
if (!countHelper) {
countHelper = std::make_unique<QDecompressHelper>();
- countHelper->setArchiveBombDetectionEnabled(archiveBombDetectionEnabled);
+ countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
countHelper->setEncoding(contentEncoding);
}
countHelper->feed(buffer);
@@ -357,13 +342,45 @@ bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer)
qsizetype QDecompressHelper::read(char *data, qsizetype maxSize)
{
+ if (maxSize <= 0)
+ return 0;
+
if (!isValid())
return -1;
- qsizetype bytesRead = -1;
if (!hasData())
return 0;
+ qsizetype cachedRead = 0;
+ if (!decompressedDataBuffer.isEmpty()) {
+ cachedRead = decompressedDataBuffer.read(data, maxSize);
+ data += cachedRead;
+ maxSize -= cachedRead;
+ }
+
+ qsizetype bytesRead = readInternal(data, maxSize);
+ if (bytesRead == -1)
+ return -1;
+ totalBytesRead += bytesRead + cachedRead;
+ return bytesRead + cachedRead;
+}
+
+/*!
+ \internal
+ Like read() but without attempting to read the
+ cached/already-decompressed data.
+*/
+qsizetype QDecompressHelper::readInternal(char *data, qsizetype maxSize)
+{
+ Q_ASSERT(isValid());
+
+ if (maxSize <= 0)
+ return 0;
+
+ if (!hasDataInternal())
+ return 0;
+
+ qsizetype bytesRead = -1;
switch (contentEncoding) {
case None:
Q_UNREACHABLE();
@@ -381,44 +398,38 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize)
}
if (bytesRead == -1)
clear();
- else if (countDecompressed)
- uncompressedBytes -= bytesRead;
totalUncompressedBytes += bytesRead;
- if (isPotentialArchiveBomb())
+ if (isPotentialArchiveBomb()) {
+ errorStr = QCoreApplication::translate(
+ "QHttp",
+ "The decompressed output exceeds the limits specified by "
+ "QNetworkRequest::decompressedSafetyCheckThreshold()");
return -1;
+ }
return bytesRead;
}
/*!
\internal
- Disables or enables checking the decompression ratio of archives
- according to the value of \a enable.
- Only for enabling us to test handling of large decompressed files
- without needing to bundle large compressed files.
+ Set the \a threshold required before the archive bomb detection kicks in.
+ By default this is 10MB. Setting it to -1 is treated as disabling the
+ feature.
*/
-void QDecompressHelper::setArchiveBombDetectionEnabled(bool enable)
-{
- archiveBombDetectionEnabled = enable;
- if (countHelper)
- countHelper->setArchiveBombDetectionEnabled(enable);
-}
-
-void QDecompressHelper::setMinimumArchiveBombSize(qint64 threshold)
+void QDecompressHelper::setDecompressedSafetyCheckThreshold(qint64 threshold)
{
- minimumArchiveBombSize = threshold;
+ if (threshold == -1)
+ threshold = std::numeric_limits<qint64>::max();
+ archiveBombCheckThreshold = threshold;
}
bool QDecompressHelper::isPotentialArchiveBomb() const
{
- if (!archiveBombDetectionEnabled)
- return false;
-
if (totalCompressedBytes == 0)
return false;
- if (totalUncompressedBytes <= minimumArchiveBombSize)
+ if (totalUncompressedBytes <= archiveBombCheckThreshold)
return false;
// Some protection against malicious or corrupted compressed files that expand far more than
@@ -430,12 +441,16 @@ bool QDecompressHelper::isPotentialArchiveBomb() const
break;
case Deflate:
case GZip:
+ // This value is mentioned in docs for
+ // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized
if (ratio > 40) {
return true;
}
break;
case Brotli:
case Zstandard:
+ // This value is mentioned in docs for
+ // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized
if (ratio > 100) {
return true;
}
@@ -454,6 +469,16 @@ bool QDecompressHelper::isPotentialArchiveBomb() const
*/
bool QDecompressHelper::hasData() const
{
+ return hasDataInternal() || !decompressedDataBuffer.isEmpty();
+}
+
+/*!
+ \internal
+ Like hasData() but internally the buffer of decompressed data is
+ not interesting.
+*/
+bool QDecompressHelper::hasDataInternal() const
+{
return encodedBytesAvailable() || decoderHasData;
}
@@ -462,11 +487,29 @@ qint64 QDecompressHelper::encodedBytesAvailable() const
return compressedDataBuffer.byteAmount();
}
+/*!
+ \internal
+ Returns whether or not the object is valid.
+ If it becomes invalid after an operation has been performed
+ then an error has occurred.
+ \sa errorString()
+*/
bool QDecompressHelper::isValid() const
{
return contentEncoding != None;
}
+/*!
+ \internal
+ Returns a string describing the error that occurred or an empty
+ string if no error occurred.
+ \sa isValid()
+*/
+QString QDecompressHelper::errorString() const
+{
+ return errorStr;
+}
+
void QDecompressHelper::clear()
{
switch (contentEncoding) {
@@ -501,13 +544,16 @@ void QDecompressHelper::clear()
contentEncoding = None;
compressedDataBuffer.clear();
+ decompressedDataBuffer.clear();
decoderHasData = false;
countDecompressed = false;
countHelper.reset();
- uncompressedBytes = 0;
+ totalBytesRead = 0;
totalUncompressedBytes = 0;
totalCompressedBytes = 0;
+
+ errorStr.clear();
}
qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize)
@@ -663,8 +709,9 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize)
switch (result) {
case BROTLI_DECODER_RESULT_ERROR:
- qWarning("Brotli error: %s",
- BrotliDecoderErrorString(BrotliDecoderGetErrorCode(brotliDecoderState)));
+ errorStr = QLatin1String("Brotli error: %1")
+ .arg(QString::fromUtf8(BrotliDecoderErrorString(
+ BrotliDecoderGetErrorCode(brotliDecoderState))));
return -1;
case BROTLI_DECODER_RESULT_SUCCESS:
BrotliDecoderDestroyInstance(brotliDecoderState);
@@ -710,7 +757,8 @@ qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize)
while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) {
size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf);
if (ZSTD_isError(retValue)) {
- qWarning("ZStandard error: %s", ZSTD_getErrorName(retValue));
+ errorStr = QLatin1String("ZStandard error: %1")
+ .arg(QString::fromUtf8(ZSTD_getErrorName(retValue)));
return -1;
} else {
decoderHasData = false;