diff options
Diffstat (limited to 'src/network/access/qnetworkreplyhttpimpl.cpp')
| -rw-r--r-- | src/network/access/qnetworkreplyhttpimpl.cpp | 515 |
1 files changed, 317 insertions, 198 deletions
diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 555b609433..1eee98f834 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtNetwork module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only //#define QNETWORKACCESSHTTPBACKEND_DEBUG @@ -57,6 +21,7 @@ #include "QtCore/qcoreapplication.h" #include <QtCore/private/qthread_p.h> +#include <QtCore/private/qtools_p.h> #include "qnetworkcookiejar.h" #include "qnetconmonitor_p.h" @@ -67,16 +32,18 @@ QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; +using namespace QtMiscUtils; +using namespace std::chrono_literals; + class QNetworkProxy; -static inline bool isSeparator(char c) -{ - static const char separators[] = "()<>@,;:\\\"/[]?={}"; - return isLWS(c) || strchr(separators, c) != nullptr; -} +static inline QByteArray rangeName() { return "Range"_ba; } +static inline QByteArray cacheControlName() { return "Cache-Control"_ba; } +static constexpr QByteArrayView bytesEqualPrefix() noexcept { return "bytes="; } // ### merge with nextField in cookiejar.cpp -static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &header) +static QHash<QByteArray, QByteArray> parseHttpOptionHeader(QByteArrayView header) { // The HTTP header is of the form: // header = #1(directives) @@ -88,7 +55,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea while (true) { // skip spaces pos = nextNonWhitespace(header, pos); - if (pos == header.length()) + if (pos == header.size()) return result; // end of parsing // pos points to a non-whitespace @@ -102,36 +69,36 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea // of the header, whichever comes first int end = comma; if (end == -1) - end = header.length(); + end = header.size(); if (equal != -1 && end > equal) end = equal; // equal sign comes before comma/end - QByteArray key = QByteArray(header.constData() + pos, end - pos).trimmed().toLower(); + const auto key = header.sliced(pos, end - pos).trimmed(); pos = end + 1; if (uint(equal) < uint(comma)) { // case: token "=" (token | quoted-string) // skip spaces pos = nextNonWhitespace(header, pos); - if (pos == header.length()) + if (pos == header.size()) // huh? Broken header return result; QByteArray value; - value.reserve(header.length() - pos); + value.reserve(header.size() - pos); if (header.at(pos) == '"') { // case: quoted-string // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) // qdtext = <any TEXT except <">> // quoted-pair = "\" CHAR ++pos; - while (pos < header.length()) { + while (pos < header.size()) { char c = header.at(pos); if (c == '"') { // end of quoted text break; } else if (c == '\\') { ++pos; - if (pos >= header.length()) + if (pos >= header.size()) // broken header return result; c = header.at(pos); @@ -141,8 +108,13 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea ++pos; } } else { + const auto isSeparator = [](char c) { + static const char separators[] = "()<>@,;:\\\"/[]?={}"; + return isLWS(c) || strchr(separators, c) != nullptr; + }; + // case: token - while (pos < header.length()) { + while (pos < header.size()) { char c = header.at(pos); if (isSeparator(c)) break; @@ -151,7 +123,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea } } - result.insert(key, value); + result.insert(key.toByteArray().toLower(), value); // find the comma now: comma = header.indexOf(',', pos); @@ -161,7 +133,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea } else { // case: token // key is already set - result.insert(key, QByteArray()); + result.insert(key.toByteArray().toLower(), QByteArray()); } } } @@ -182,10 +154,13 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage d->outgoingData = outgoingData; d->url = request.url(); #ifndef QT_NO_SSL - if (request.url().scheme() == QLatin1String("https")) + if (request.url().scheme() == "https"_L1) d->sslConfiguration.reset(new QSslConfiguration(request.sslConfiguration())); #endif + QObjectPrivate::connect(this, &QNetworkReplyHttpImpl::redirectAllowed, d, + &QNetworkReplyHttpImplPrivate::followRedirect, Qt::QueuedConnection); + // FIXME Later maybe set to Unbuffered, especially if it is zerocopy or from cache? QIODevice::open(QIODevice::ReadOnly); @@ -199,7 +174,7 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage if (d->synchronous && outgoingData) { // The synchronous HTTP is a corner case, we will put all upload data in one big QByteArray in the outgoingDataBuffer. // Yes, this is not the most efficient thing to do, but on the other hand synchronous XHR needs to die anyway. - d->outgoingDataBuffer = QSharedPointer<QRingBuffer>::create(); + d->outgoingDataBuffer = std::make_shared<QRingBuffer>(); qint64 previousDataSize = 0; do { previousDataSize = d->outgoingDataBuffer->size(); @@ -305,6 +280,13 @@ qint64 QNetworkReplyHttpImpl::bytesAvailable() const return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition; } + if (d->decompressHelper.isValid()) { + if (d->decompressHelper.isCountingBytes()) + return QNetworkReply::bytesAvailable() + d->decompressHelper.uncompressedSize(); + if (d->decompressHelper.hasData()) + return QNetworkReply::bytesAvailable() + 1; + } + // normal buffer return QNetworkReply::bytesAvailable(); } @@ -345,6 +327,32 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) } + if (d->decompressHelper.isValid() && (d->decompressHelper.hasData() || !isFinished())) { + if (maxlen == 0 || !d->decompressHelper.hasData()) + return 0; + const qint64 bytesRead = d->decompressHelper.read(data, maxlen); + if (!d->decompressHelper.isValid()) { + d->error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(d->decompressHelper.errorString())); + d->decompressHelper.clear(); + return -1; + } + if (d->cacheSaveDevice) { + // Need to write to the cache now that we have the data + d->cacheSaveDevice->write(data, bytesRead); + // ... and if we've read everything then the cache can be closed. + if (isFinished() && !d->decompressHelper.hasData()) + d->completeCacheSave(); + } + // In case of buffer size restriction we need to emit that it has been emptied + qint64 wasBuffered = d->bytesBuffered; + d->bytesBuffered = 0; + if (readBufferSize()) + emit readBufferFreed(wasBuffered); + return bytesRead; + } + // normal buffer if (d->state == d->Finished || d->state == d->Aborted) return -1; @@ -447,8 +455,8 @@ QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() , downloadBufferReadPosition(0) , downloadBufferCurrentSize(0) , downloadZerocopyBuffer(nullptr) - , pendingDownloadDataEmissions(QSharedPointer<QAtomicInt>::create()) - , pendingDownloadProgressEmissions(QSharedPointer<QAtomicInt>::create()) + , pendingDownloadDataEmissions(std::make_shared<QAtomicInt>()) + , pendingDownloadProgressEmissions(std::make_shared<QAtomicInt>()) #ifndef QT_NO_SSL , pendingIgnoreAllSslErrors(false) #endif @@ -473,16 +481,17 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { // If the request does not already specify preferred cache-control // force reload from the network and tell any caching proxy servers to reload too - if (!request.rawHeaderList().contains("Cache-Control")) { - httpRequest.setHeaderField("Cache-Control", "no-cache"); - httpRequest.setHeaderField("Pragma", "no-cache"); + if (!request.rawHeaderList().contains(cacheControlName())) { + const auto noCache = "no-cache"_ba; + httpRequest.setHeaderField(cacheControlName(), noCache); + httpRequest.setHeaderField("Pragma"_ba, noCache); } return false; } // The disk cache API does not currently support partial content retrieval. // That is why we don't use the disk cache for any such requests. - if (request.hasRawHeader("Range")) + if (request.hasRawHeader(rangeName())) return false; QAbstractNetworkCache *nc = managerPrivate->networkCache; @@ -500,20 +509,27 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); + it = cacheHeaders.findRawHeader("content-length"); + if (it != cacheHeaders.rawHeaders.constEnd()) { + std::unique_ptr<QIODevice> data(nc->data(httpRequest.url())); + if (!data || data->size() < it->second.toLongLong()) + return false; // The data is smaller than the content-length specified + } + it = cacheHeaders.findRawHeader("etag"); if (it != cacheHeaders.rawHeaders.constEnd()) - httpRequest.setHeaderField("If-None-Match", it->second); + httpRequest.setHeaderField("If-None-Match"_ba, it->second); QDateTime lastModified = metaData.lastModified(); if (lastModified.isValid()) - httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); + httpRequest.setHeaderField("If-Modified-Since"_ba, QNetworkHeadersPrivate::toHttpDate(lastModified)); - it = cacheHeaders.findRawHeader("Cache-Control"); + it = cacheHeaders.findRawHeader(cacheControlName()); if (it != cacheHeaders.rawHeaders.constEnd()) { QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) + if (cacheControl.contains("must-revalidate"_ba)) return false; - if (cacheControl.contains("no-cache")) + if (cacheControl.contains("no-cache"_ba)) return false; } @@ -568,10 +584,11 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (lastModified.isValid() && dateHeader.isValid()) { qint64 diff = lastModified.secsTo(dateHeader); freshness_lifetime = diff / 10; - if (httpRequest.headerField("Warning").isEmpty()) { + const auto warningHeader = "Warning"_ba; + if (httpRequest.headerField(warningHeader).isEmpty()) { QDateTime dt = currentDateTime.addSecs(current_age); if (currentDateTime.daysTo(dt) > 1) - httpRequest.setHeaderField("Warning", "113"); + httpRequest.setHeaderField(warningHeader, "113"_ba); } } @@ -627,13 +644,11 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq httpRequest.setRedirectCount(newHttpRequest.maximumRedirectsAllowed()); QString scheme = url.scheme(); - bool ssl = (scheme == QLatin1String("https") - || scheme == QLatin1String("preconnect-https")); + bool ssl = (scheme == "https"_L1 || scheme == "preconnect-https"_L1); q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); httpRequest.setSsl(ssl); - bool preConnect = (scheme == QLatin1String("preconnect-http") - || scheme == QLatin1String("preconnect-https")); + bool preConnect = (scheme == "preconnect-http"_L1 || scheme == "preconnect-https"_L1); httpRequest.setPreConnect(preConnect); #ifndef QT_NO_NETWORKPROXY @@ -680,12 +695,18 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq httpRequest.setRedirectPolicy(redirectPolicy); httpRequest.setPriority(convert(newHttpRequest.priority())); + loadingFromCache = false; switch (operation) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); - if (loadFromCacheIfAllowed(httpRequest)) + // If the request has a body, createUploadByteDevice() and don't use caching + if (outgoingData) { + invalidateCache(); + createUploadByteDevice(); + } else if (loadFromCacheIfAllowed(httpRequest)) { return; // no need to send the request! :) + } break; case QNetworkAccessManager::HeadOperation: @@ -725,14 +746,15 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq QList<QByteArray> headers = newHttpRequest.rawHeaderList(); if (resumeOffset != 0) { - const int rangeIndex = headers.indexOf("Range"); + const int rangeIndex = headers.indexOf(rangeName()); if (rangeIndex != -1) { // Need to adjust resume offset for user specified range headers.removeAt(rangeIndex); // We've already verified that requestRange starts with "bytes=", see canResume. - QByteArray requestRange = newHttpRequest.rawHeader("Range").mid(6); + const auto rangeHeader = newHttpRequest.rawHeader(rangeName()); + const auto requestRange = QByteArrayView(rangeHeader).mid(bytesEqualPrefix().size()); int index = requestRange.indexOf('-'); @@ -740,16 +762,16 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq quint64 requestEndOffset = requestRange.mid(index + 1).toULongLong(); // In case an end offset is not given it is skipped from the request range - requestRange = "bytes=" + QByteArray::number(resumeOffset + requestStartOffset) + + QByteArray newRange = bytesEqualPrefix() + QByteArray::number(resumeOffset + requestStartOffset) + '-' + (requestEndOffset ? QByteArray::number(requestEndOffset) : QByteArray()); - httpRequest.setHeaderField("Range", requestRange); + httpRequest.setHeaderField(rangeName(), newRange); } else { - httpRequest.setHeaderField("Range", "bytes=" + QByteArray::number(resumeOffset) + '-'); + httpRequest.setHeaderField(rangeName(), bytesEqualPrefix() + QByteArray::number(resumeOffset) + '-'); } } - for (const QByteArray &header : qAsConst(headers)) + for (const QByteArray &header : std::as_const(headers)) httpRequest.setHeaderField(header, newHttpRequest.rawHeader(header)); if (newHttpRequest.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool()) @@ -759,6 +781,12 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq allowed.isValid() && allowed.canConvert<bool>()) { httpRequest.setHTTP2Allowed(allowed.value<bool>()); } + auto h2cAttribute = request.attribute(QNetworkRequest::Http2CleartextAllowedAttribute); + // ### Qt7: Stop checking the environment variable + if (h2cAttribute.toBool() + || (!h2cAttribute.isValid() && qEnvironmentVariableIsSet("QT_NETWORK_H2C_ALLOWED"))) { + httpRequest.setH2cAllowed(true); + } if (request.attribute(QNetworkRequest::Http2DirectAttribute).toBool()) { // Intentionally mutually exclusive - cannot be both direct and 'allowed' @@ -774,20 +802,16 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq if (request.attribute(QNetworkRequest::EmitAllUploadProgressSignalsAttribute).toBool()) emitAllUploadProgressSignals = true; - // For internal use/testing - auto ignoreDownloadRatio = - request.attribute(QNetworkRequest::Attribute(QNetworkRequest::User - 1)); - if (!ignoreDownloadRatio.isNull() && ignoreDownloadRatio.canConvert<QByteArray>() - && ignoreDownloadRatio.toByteArray() == "__qdecompresshelper_ignore_download_ratio") { - httpRequest.setIgnoreDecompressionRatio(true); - } - httpRequest.setPeerVerifyName(newHttpRequest.peerVerifyName()); // Create the HTTP thread delegate QHttpThreadDelegate *delegate = new QHttpThreadDelegate; // Propagate Http/2 settings: delegate->http2Parameters = request.http2Configuration(); + delegate->http1Parameters = request.http1Configuration(); + + if (request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).isValid()) + delegate->connectionCacheExpiryTimeoutSeconds = request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).toInt(); // For the synchronous HTTP, this is the normal way the delegate gets deleted // For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished @@ -846,14 +870,12 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq QObject::connect(delegate, SIGNAL(downloadFinished()), q, SLOT(replyFinished()), Qt::QueuedConnection); - QObject::connect(delegate, SIGNAL(downloadMetaData(QList<QPair<QByteArray,QByteArray> >, - int, QString, bool, - QSharedPointer<char>, qint64, qint64, - bool)), - q, SLOT(replyDownloadMetaData(QList<QPair<QByteArray,QByteArray> >, - int, QString, bool, - QSharedPointer<char>, qint64, qint64, bool)), - Qt::QueuedConnection); + QObject::connect(delegate, &QHttpThreadDelegate::socketStartedConnecting, + q, &QNetworkReply::socketStartedConnecting, Qt::QueuedConnection); + QObject::connect(delegate, &QHttpThreadDelegate::requestSent, + q, &QNetworkReply::requestSent, Qt::QueuedConnection); + connect(delegate, &QHttpThreadDelegate::downloadMetaData, this, + &QNetworkReplyHttpImplPrivate::replyDownloadMetaData, Qt::QueuedConnection); QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)), q, SLOT(replyDownloadProgressSlot(qint64,qint64)), Qt::QueuedConnection); @@ -864,9 +886,6 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq q, SLOT(onRedirected(QUrl,int,int)), Qt::QueuedConnection); - QObject::connect(q, SIGNAL(redirectAllowed()), q, SLOT(followRedirect()), - Qt::QueuedConnection); - #ifndef QT_NO_SSL QObject::connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), q, SLOT(replySslConfigurationChanged(QSslConfiguration)), @@ -906,14 +925,14 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq delegate->httpRequest.setUploadByteDevice(forwardUploadDevice); // If the device in the user thread claims it has more data, keep the flow to HTTP thread going - QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), + QObject::connect(uploadByteDevice.get(), SIGNAL(readyRead()), q, SLOT(uploadByteDeviceReadyReadSlot()), Qt::QueuedConnection); // From user thread to http thread: QObject::connect(q, SIGNAL(haveUploadData(qint64,QByteArray,bool,qint64)), forwardUploadDevice, SLOT(haveDataSlot(qint64,QByteArray,bool,qint64)), Qt::QueuedConnection); - QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), + QObject::connect(uploadByteDevice.get(), SIGNAL(readyRead()), forwardUploadDevice, SIGNAL(readyRead()), Qt::QueuedConnection); @@ -936,7 +955,7 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq // use the uploadByteDevice provided to us by the QNetworkReplyImpl. // The code that is in start() makes sure it is safe to use from a thread // since it only wraps a QRingBuffer - delegate->httpRequest.setUploadByteDevice(uploadByteDevice.data()); + delegate->httpRequest.setUploadByteDevice(uploadByteDevice.get()); } } @@ -953,30 +972,20 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq if (synchronous) { emit q->startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done - if (delegate->incomingErrorCode != QNetworkReply::NoError) { - replyDownloadMetaData - (delegate->incomingHeaders, - delegate->incomingStatusCode, - delegate->incomingReasonPhrase, - delegate->isPipeliningUsed, - QSharedPointer<char>(), - delegate->incomingContentLength, - delegate->removedContentLength, - delegate->isHttp2Used); - replyDownloadData(delegate->synchronousDownloadData); + replyDownloadMetaData + (delegate->incomingHeaders, + delegate->incomingStatusCode, + delegate->incomingReasonPhrase, + delegate->isPipeliningUsed, + QSharedPointer<char>(), + delegate->incomingContentLength, + delegate->removedContentLength, + delegate->isHttp2Used, + delegate->isCompressed); + replyDownloadData(delegate->synchronousDownloadData); + + if (delegate->incomingErrorCode != QNetworkReply::NoError) httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail); - } else { - replyDownloadMetaData - (delegate->incomingHeaders, - delegate->incomingStatusCode, - delegate->incomingReasonPhrase, - delegate->isPipeliningUsed, - QSharedPointer<char>(), - delegate->incomingContentLength, - delegate->removedContentLength, - delegate->isHttp2Used); - replyDownloadData(delegate->synchronousDownloadData); - } thread->quit(); thread->wait(QDeadlineTimer(5000)); @@ -1047,23 +1056,79 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) if (!q->isOpen()) return; + // cache this, we need it later and it's invalidated when dealing with compressed data + auto dataSize = d.size(); + if (cacheEnabled && isCachingAllowed() && !cacheSaveDevice) initCacheSaveDevice(); + if (decompressHelper.isValid()) { + qint64 uncompressedBefore = -1; + if (decompressHelper.isCountingBytes()) + uncompressedBefore = decompressHelper.uncompressedSize(); + + decompressHelper.feed(std::move(d)); + + if (!decompressHelper.isValid()) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(decompressHelper.errorString())); + decompressHelper.clear(); + return; + } + + if (!isHttpRedirectResponse()) { + if (decompressHelper.isCountingBytes()) + bytesDownloaded += (decompressHelper.uncompressedSize() - uncompressedBefore); + setupTransferTimeout(); + } + + if (synchronous) { + d = QByteArray(); + const qsizetype increments = 16 * 1024; + qint64 bytesRead = 0; + while (decompressHelper.hasData()) { + quint64 nextSize = quint64(d.size()) + quint64(increments); + if (nextSize > quint64(std::numeric_limits<QByteArray::size_type>::max())) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", + "Data downloaded is too large to store")); + decompressHelper.clear(); + return; + } + d.resize(nextSize); + bytesRead += decompressHelper.read(d.data() + bytesRead, increments); + if (!decompressHelper.isValid()) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(decompressHelper.errorString())); + decompressHelper.clear(); + return; + } + } + d.resize(bytesRead); + // we're synchronous so we're not calling this function again; reset the decompressHelper + decompressHelper.clear(); + } + } + // This is going to look a little strange. When downloading data while a // HTTP redirect is happening (and enabled), we write the redirect // response to the cache. However, we do not append it to our internal // buffer as that will contain the response data only for the final // response - if (cacheSaveDevice) + // Note: For compressed data this is done in readData() + if (cacheSaveDevice && !decompressHelper.isValid()) { cacheSaveDevice->write(d); + } - if (!isHttpRedirectResponse()) { + // if decompressHelper is valid then we have compressed data, and this is handled above + if (!decompressHelper.isValid() && !isHttpRedirectResponse()) { buffer.append(d); - bytesDownloaded += d.size(); + bytesDownloaded += dataSize; setupTransferTimeout(); } - bytesBuffered += d.size(); + bytesBuffered += dataSize; int pendingSignals = pendingDownloadDataEmissions->fetchAndSubAcquire(1) - 1; if (pendingSignals > 0) { @@ -1077,17 +1142,26 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) if (isHttpRedirectResponse()) return; + // This can occur when downloading compressed data as some of the data may be the content + // encoding's header. Don't emit anything for this. + if (lastReadyReadEmittedSize == bytesDownloaded) { + if (readBufferMaxSize) + emit q->readBufferFreed(dataSize); + return; + } + lastReadyReadEmittedSize = bytesDownloaded; + QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); emit q->readyRead(); - // emit readyRead before downloadProgress incase this will cause events to be + // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). - if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) { + if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval + && (!decompressHelper.isValid() || decompressHelper.isCountingBytes())) { downloadProgressSignalChoke.restart(); emit q->downloadProgress(bytesDownloaded, totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); } - } void QNetworkReplyHttpImplPrivate::replyFinished() @@ -1158,13 +1232,12 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt // equal to "80", the port component value MUST be preserved; // otherwise, if the URI does not contain an explicit port // component, the UA MUST NOT add one. - url.setScheme(QLatin1String("https")); + url.setScheme("https"_L1); if (url.port() == 80) url.setPort(443); } - const bool isLessSafe = schemeBefore == QLatin1String("https") - && url.scheme() == QLatin1String("http"); + const bool isLessSafe = schemeBefore == "https"_L1 && url.scheme() == "http"_L1; if (httpRequest.redirectPolicy() == QNetworkRequest::NoLessSafeRedirectPolicy && isLessSafe) { error(QNetworkReply::InsecureRedirectError, @@ -1172,13 +1245,18 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt return; } + // If the original operation was a GET with a body and the status code is either + // 307 or 308 then keep the message body + const bool getOperationKeepsBody = (operation == QNetworkAccessManager::GetOperation) + && (httpStatus == 307 || httpStatus == 308); + redirectRequest = createRedirectRequest(originalRequest, url, maxRedirectsRemaining); operation = getRedirectOperation(operation, httpStatus); // Clear stale headers, the relevant ones get set again later httpRequest.clearHeaders(); - if (operation == QNetworkAccessManager::GetOperation - || operation == QNetworkAccessManager::HeadOperation) { + if ((operation == QNetworkAccessManager::GetOperation + || operation == QNetworkAccessManager::HeadOperation) && !getOperationKeepsBody) { // possibly changed from not-GET/HEAD to GET/HEAD, make sure to get rid of upload device uploadByteDevice.reset(); uploadByteDevicePosition = 0; @@ -1214,6 +1292,7 @@ void QNetworkReplyHttpImplPrivate::followRedirect() Q_Q(QNetworkReplyHttpImpl); Q_ASSERT(managerPrivate); + decompressHelper.clear(); rawHeaders.clear(); cookedHeaders.clear(); @@ -1224,6 +1303,8 @@ void QNetworkReplyHttpImplPrivate::followRedirect() q, [this]() { postRequest(redirectRequest); }, Qt::QueuedConnection); } +static constexpr QLatin1StringView locationHeader() noexcept { return "location"_L1; } + void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) { Q_Q(QNetworkReplyHttpImpl); @@ -1236,20 +1317,20 @@ void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) // What do we do about the caching of the HTML note? // The response to a 303 MUST NOT be cached, while the response to // all of the others is cacheable if the headers indicate it to be - QByteArray header = q->rawHeader("location"); + QByteArray header = q->rawHeader(locationHeader()); QUrl url = QUrl(QString::fromUtf8(header)); if (!url.isValid()) - url = QUrl(QLatin1String(header)); + url = QUrl(QLatin1StringView(header)); q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url); } } -void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByteArray,QByteArray> > &hm, +void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QHttpHeaders &hm, int sc, const QString &rp, bool pu, QSharedPointer<char> db, qint64 contentLength, qint64 removedContentLength, - bool h2Used) + bool h2Used, bool isCompressed) { Q_Q(QNetworkReplyHttpImpl); Q_UNUSED(contentLength); @@ -1263,7 +1344,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte // RFC6797, 8.1 // If an HTTP response is received over insecure transport, the UA MUST // ignore any present STS header field(s). - if (url.scheme() == QLatin1String("https") && managerPrivate->stsEnabled) + if (url.scheme() == "https"_L1 && managerPrivate->stsEnabled) managerPrivate->stsCache.updateFromHeaders(hm, url); #endif // Download buffer @@ -1277,29 +1358,48 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte q->setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); q->setAttribute(QNetworkRequest::Http2WasUsedAttribute, h2Used); + // A user having manually defined which encodings they accept is, for + // somwehat unknown (presumed legacy compatibility) reasons treated as + // disabling our decompression: + const bool autoDecompress = request.rawHeader("accept-encoding").isEmpty(); + const bool shouldDecompress = isCompressed && autoDecompress; // reconstruct the HTTP header - QList<QPair<QByteArray, QByteArray> > headerMap = hm; - QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(), - end = headerMap.constEnd(); - for (; it != end; ++it) { - QByteArray value = q->rawHeader(it->first); + for (qsizetype i = 0; i < hm.size(); ++i) { + const auto key = hm.nameAt(i); + const auto originValue = hm.valueAt(i); + + QByteArray value = q->rawHeader(key); // Reset any previous "location" header set in the reply. In case of // redirects, we don't want to 'append' multiple location header values, // rather we keep only the latest one - if (it->first.toLower() == "location") + if (key == locationHeader()) value.clear(); + if (shouldDecompress && !decompressHelper.isValid() && key == "content-encoding"_L1) { + if (!synchronous) // with synchronous all the data is expected to be handled at once + decompressHelper.setCountingBytesEnabled(true); + + if (!decompressHelper.setEncoding(originValue)) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Failed to initialize decompression: %1") + .arg(decompressHelper.errorString())); + return; + } + decompressHelper.setDecompressedSafetyCheckThreshold( + request.decompressedSafetyCheckThreshold()); + } + if (!value.isEmpty()) { // Why are we appending values for headers which are already // present? - if (it->first.compare("set-cookie", Qt::CaseInsensitive) == 0) + if (key == "set-cookie"_L1) value += '\n'; else value += ", "; } - value += it->second; - q->setRawHeader(it->first, value); + value += originValue; + q->setRawHeader({key.data(), key.size()}, value); } q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); @@ -1318,11 +1418,11 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte QNetworkHeadersPrivate cacheHeaders; cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - it = cacheHeaders.findRawHeader("Cache-Control"); + it = cacheHeaders.findRawHeader(cacheControlName()); bool mustReValidate = false; if (it != cacheHeaders.rawHeaders.constEnd()) { QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) + if (cacheControl.contains("must-revalidate"_ba)) mustReValidate = true; } if (!mustReValidate && sendCacheContents(metaData)) @@ -1392,7 +1492,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadProgressSlot(qint64 bytesReceive downloadBufferCurrentSize = bytesReceived; // Only emit readyRead when actual data is there - // emit readyRead before downloadProgress incase this will cause events to be + // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). if (bytesDownloaded > 0) emit q->readyRead(); @@ -1566,7 +1666,7 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData QUrl redirectUrl; for ( ; it != end; ++it) { if (httpRequest.isFollowRedirects() && - !it->first.compare("location", Qt::CaseInsensitive)) + !it->first.compare(locationHeader(), Qt::CaseInsensitive)) redirectUrl = QUrl::fromEncoded(it->second); setRawHeader(it->first, it->second); } @@ -1603,6 +1703,27 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData return true; } +static auto caseInsensitiveCompare(QByteArrayView value) +{ + return [value](QByteArrayView element) + { + return value.compare(element, Qt::CaseInsensitive) == 0; + }; +} + +static bool isHopByHop(QByteArrayView header) +{ + constexpr QByteArrayView headers[] = { "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade"}; + return std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(header)); +} + QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const { Q_Q(const QNetworkReplyHttpImpl); @@ -1614,22 +1735,11 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; const QList<QByteArray> newHeaders = q->rawHeaderList(); - for (QByteArray header : newHeaders) { - QByteArray originalHeader = header; - header = header.toLower(); - bool hop_by_hop = - (header == "connection" - || header == "keep-alive" - || header == "proxy-authenticate" - || header == "proxy-authorization" - || header == "te" - || header == "trailers" - || header == "transfer-encoding" - || header == "upgrade"); - if (hop_by_hop) + for (const QByteArray& header : newHeaders) { + if (isHopByHop(header)) continue; - if (header == "set-cookie") + if (header.compare("set-cookie", Qt::CaseInsensitive) == 0) continue; // for 4.6.0, we were planning to not store the date header in the @@ -1642,27 +1752,27 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe //continue; // Don't store Warning 1xx headers - if (header == "warning") { - QByteArray v = q->rawHeader(header); - if (v.length() == 3 + if (header.compare("warning", Qt::CaseInsensitive) == 0) { + const QByteArray v = q->rawHeader(header); + if (v.size() == 3 && v[0] == '1' - && v[1] >= '0' && v[1] <= '9' - && v[2] >= '0' && v[2] <= '9') + && isAsciiDigit(v[1]) + && isAsciiDigit(v[2])) continue; } it = cacheHeaders.findRawHeader(header); if (it != cacheHeaders.rawHeaders.constEnd()) { // Match the behavior of Firefox and assume Cache-Control: "no-transform" - if (header == "content-encoding" - || header == "content-range" - || header == "content-type") + constexpr QByteArrayView headers[]= + {"content-encoding", "content-range", "content-type"}; + if (std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(header))) continue; } // IIS has been known to send "Content-Length: 0" on 304 responses, so // ignore this too - if (header == "content-length" && statusCode == 304) + if (statusCode == 304 && header.compare("content-length", Qt::CaseInsensitive) == 0) continue; #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) @@ -1670,23 +1780,23 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe QByteArray o; if (it != cacheHeaders.rawHeaders.constEnd()) o = (*it).second; - if (n != o && header != "date") { + if (n != o && headerheader.compare("date", Qt::CaseInsensitive) != 0) { qDebug() << "replacing" << header; qDebug() << "new" << n; qDebug() << "old" << o; } #endif - cacheHeaders.setRawHeader(originalHeader, q->rawHeader(header)); + cacheHeaders.setRawHeader(header, q->rawHeader(header)); } metaData.setRawHeaders(cacheHeaders.rawHeaders); bool checkExpired = true; QHash<QByteArray, QByteArray> cacheControl; - it = cacheHeaders.findRawHeader("Cache-Control"); + it = cacheHeaders.findRawHeader(cacheControlName()); if (it != cacheHeaders.rawHeaders.constEnd()) { cacheControl = parseHttpOptionHeader(it->second); - QByteArray maxAge = cacheControl.value("max-age"); + QByteArray maxAge = cacheControl.value("max-age"_ba); if (!maxAge.isEmpty()) { checkExpired = false; QDateTime dt = QDateTime::currentDateTimeUtc(); @@ -1713,7 +1823,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe canDiskCache = true; // HTTP/1.1. Check the Cache-Control header - if (cacheControl.contains("no-store")) + if (cacheControl.contains("no-store"_ba)) canDiskCache = false; // responses to POST might be cacheable @@ -1722,7 +1832,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe canDiskCache = false; // some pages contain "expires:" and "cache-control: no-cache" field, // so we only might cache POST requests if we get "cache-control: max-age ..." - if (cacheControl.contains("max-age")) + if (cacheControl.contains("max-age"_ba)) canDiskCache = true; // responses to PUT and DELETE are not cacheable @@ -1753,14 +1863,14 @@ bool QNetworkReplyHttpImplPrivate::canResume() const return false; // Can only resume if server/resource supports Range header. - QByteArray acceptRangesheaderName("Accept-Ranges"); + constexpr auto acceptRangesheaderName = QByteArrayView("Accept-Ranges"); if (!q->hasRawHeader(acceptRangesheaderName) || q->rawHeader(acceptRangesheaderName) == "none") return false; // We only support resuming for byte ranges. - if (request.hasRawHeader("Range")) { - QByteArray range = request.rawHeader("Range"); - if (!range.startsWith("bytes=")) + if (request.hasRawHeader(rangeName())) { + QByteArray range = request.rawHeader(rangeName()); + if (!range.startsWith(bytesEqualPrefix())) return false; } @@ -1779,7 +1889,9 @@ void QNetworkReplyHttpImplPrivate::setResumeOffset(quint64 offset) void QNetworkReplyHttpImplPrivate::_q_startOperation() { - if (state == Working) // ensure this function is only being called once + // Ensure this function is only being called once, and not at all if we were + // cancelled + if (state >= Working) return; state = Working; @@ -1809,7 +1921,7 @@ void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead() QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); - // emit readyRead before downloadProgress incase this will cause events to be + // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). if (!(isHttpRedirectResponse())) { @@ -1886,7 +1998,7 @@ void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingData() if (!outgoingDataBuffer) { // first call, create our buffer - outgoingDataBuffer = QSharedPointer<QRingBuffer>::create(); + outgoingDataBuffer = std::make_shared<QRingBuffer>(); QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); @@ -1939,9 +2051,9 @@ void QNetworkReplyHttpImplPrivate::setupTransferTimeout() Qt::QueuedConnection); } transferTimeout->stop(); - if (request.transferTimeout()) { + if (request.transferTimeoutAsDuration() > 0ms) { transferTimeout->setSingleShot(true); - transferTimeout->setInterval(request.transferTimeout()); + transferTimeout->setInterval(request.transferTimeoutAsDuration()); QMetaObject::invokeMethod(transferTimeout, "start", Qt::QueuedConnection); @@ -1986,10 +2098,10 @@ QNonContiguousByteDevice* QNetworkReplyHttpImplPrivate::createUploadByteDevice() // We want signal emissions only for normal asynchronous uploads if (!synchronous) - QObject::connect(uploadByteDevice.data(), SIGNAL(readProgress(qint64,qint64)), + QObject::connect(uploadByteDevice.get(), SIGNAL(readProgress(qint64,qint64)), q, SLOT(emitReplyUploadProgress(qint64,qint64))); - return uploadByteDevice.data(); + return uploadByteDevice.get(); } void QNetworkReplyHttpImplPrivate::_q_finished() @@ -2008,9 +2120,12 @@ void QNetworkReplyHttpImplPrivate::finished() QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); - // if we don't know the total size of or we received everything save the cache - if (totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize) + // if we don't know the total size of or we received everything save the cache. + // If the data is compressed then this is done in readData() + if ((totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize) + && !decompressHelper.isValid()) { completeCacheSave(); + } // We check for errorCode too as in case of SSL handshake failure, we still // get the HTTP redirect status code (301, 303 etc) @@ -2044,7 +2159,9 @@ void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, c Q_Q(QNetworkReplyHttpImpl); // Can't set and emit multiple errors. if (errorCode != QNetworkReply::NoError) { - qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."); + // But somewhat unavoidable if we have cancelled the request: + if (errorCode != QNetworkReply::OperationCanceledError) + qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."); return; } @@ -2136,3 +2253,5 @@ void QNetworkReplyHttpImplPrivate::completeCacheSave() } QT_END_NAMESPACE + +#include "moc_qnetworkreplyhttpimpl_p.cpp" |