diff options
Diffstat (limited to 'src/network/access/qnetworkreplyhttpimpl.cpp')
-rw-r--r-- | src/network/access/qnetworkreplyhttpimpl.cpp | 699 |
1 files changed, 420 insertions, 279 deletions
diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index f189f5be20..3e1fe761ee 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtNetwork module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only //#define QNETWORKACCESSHTTPBACKEND_DEBUG @@ -57,24 +21,29 @@ #include "QtCore/qcoreapplication.h" #include <QtCore/private/qthread_p.h> +#include <QtCore/private/qtools_p.h> #include "qnetworkcookiejar.h" #include "qnetconmonitor_p.h" +#include "qnetworkreplyimpl_p.h" + #include <string.h> // for strchr QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; +using namespace QtMiscUtils; +using namespace std::chrono_literals; + class QNetworkProxy; -static inline bool isSeparator(char c) -{ - static const char separators[] = "()<>@,;:\\\"/[]?={}"; - return isLWS(c) || strchr(separators, c) != nullptr; -} +static inline QByteArray rangeName() { return "Range"_ba; } +static inline QByteArray cacheControlName() { return "Cache-Control"_ba; } +static constexpr QByteArrayView bytesEqualPrefix() noexcept { return "bytes="; } // ### merge with nextField in cookiejar.cpp -static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &header) +static QHash<QByteArray, QByteArray> parseHttpOptionHeader(QByteArrayView header) { // The HTTP header is of the form: // header = #1(directives) @@ -86,7 +55,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea while (true) { // skip spaces pos = nextNonWhitespace(header, pos); - if (pos == header.length()) + if (pos == header.size()) return result; // end of parsing // pos points to a non-whitespace @@ -100,36 +69,36 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea // of the header, whichever comes first int end = comma; if (end == -1) - end = header.length(); + end = header.size(); if (equal != -1 && end > equal) end = equal; // equal sign comes before comma/end - QByteArray key = QByteArray(header.constData() + pos, end - pos).trimmed().toLower(); + const auto key = header.sliced(pos, end - pos).trimmed(); pos = end + 1; if (uint(equal) < uint(comma)) { // case: token "=" (token | quoted-string) // skip spaces pos = nextNonWhitespace(header, pos); - if (pos == header.length()) + if (pos == header.size()) // huh? Broken header return result; QByteArray value; - value.reserve(header.length() - pos); + value.reserve(header.size() - pos); if (header.at(pos) == '"') { // case: quoted-string // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) // qdtext = <any TEXT except <">> // quoted-pair = "\" CHAR ++pos; - while (pos < header.length()) { + while (pos < header.size()) { char c = header.at(pos); if (c == '"') { // end of quoted text break; } else if (c == '\\') { ++pos; - if (pos >= header.length()) + if (pos >= header.size()) // broken header return result; c = header.at(pos); @@ -139,8 +108,13 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea ++pos; } } else { + const auto isSeparator = [](char c) { + static const char separators[] = "()<>@,;:\\\"/[]?={}"; + return isLWS(c) || strchr(separators, c) != nullptr; + }; + // case: token - while (pos < header.length()) { + while (pos < header.size()) { char c = header.at(pos); if (isSeparator(c)) break; @@ -149,7 +123,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea } } - result.insert(key, value); + result.insert(key.toByteArray().toLower(), value); // find the comma now: comma = header.indexOf(',', pos); @@ -159,7 +133,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea } else { // case: token // key is already set - result.insert(key, QByteArray()); + result.insert(key.toByteArray().toLower(), QByteArray()); } } } @@ -180,10 +154,13 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage d->outgoingData = outgoingData; d->url = request.url(); #ifndef QT_NO_SSL - if (request.url().scheme() == QLatin1String("https")) + if (request.url().scheme() == "https"_L1) d->sslConfiguration.reset(new QSslConfiguration(request.sslConfiguration())); #endif + QObjectPrivate::connect(this, &QNetworkReplyHttpImpl::redirectAllowed, d, + &QNetworkReplyHttpImplPrivate::followRedirect, Qt::QueuedConnection); + // FIXME Later maybe set to Unbuffered, especially if it is zerocopy or from cache? QIODevice::open(QIODevice::ReadOnly); @@ -197,7 +174,7 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage if (d->synchronous && outgoingData) { // The synchronous HTTP is a corner case, we will put all upload data in one big QByteArray in the outgoingDataBuffer. // Yes, this is not the most efficient thing to do, but on the other hand synchronous XHR needs to die anyway. - d->outgoingDataBuffer = QSharedPointer<QRingBuffer>::create(); + d->outgoingDataBuffer = std::make_shared<QRingBuffer>(); qint64 previousDataSize = 0; do { previousDataSize = d->outgoingDataBuffer->size(); @@ -225,7 +202,10 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage if (bufferingDisallowed) { // if a valid content-length header for the request was supplied, we can disable buffering // if not, we will buffer anyway - if (request.header(QNetworkRequest::ContentLengthHeader).isValid()) { + + const auto sizeOpt = QNetworkHeadersPrivate::toInt( + request.headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); + if (sizeOpt) { QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection); // FIXME make direct call? } else { @@ -303,6 +283,13 @@ qint64 QNetworkReplyHttpImpl::bytesAvailable() const return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition; } + if (d->decompressHelper.isValid()) { + if (d->decompressHelper.isCountingBytes()) + return QNetworkReply::bytesAvailable() + d->decompressHelper.uncompressedSize(); + if (d->decompressHelper.hasData()) + return QNetworkReply::bytesAvailable() + 1; + } + // normal buffer return QNetworkReply::bytesAvailable(); } @@ -343,6 +330,32 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) } + if (d->decompressHelper.isValid() && (d->decompressHelper.hasData() || !isFinished())) { + if (maxlen == 0 || !d->decompressHelper.hasData()) + return 0; + const qint64 bytesRead = d->decompressHelper.read(data, maxlen); + if (!d->decompressHelper.isValid()) { + d->error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(d->decompressHelper.errorString())); + d->decompressHelper.clear(); + return -1; + } + if (d->cacheSaveDevice) { + // Need to write to the cache now that we have the data + d->cacheSaveDevice->write(data, bytesRead); + // ... and if we've read everything then the cache can be closed. + if (isFinished() && !d->decompressHelper.hasData()) + d->completeCacheSave(); + } + // In case of buffer size restriction we need to emit that it has been emptied + qint64 wasBuffered = d->bytesBuffered; + d->bytesBuffered = 0; + if (readBufferSize()) + emit readBufferFreed(wasBuffered); + return bytesRead; + } + // normal buffer if (d->state == d->Finished || d->state == d->Aborted) return -1; @@ -445,8 +458,8 @@ QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() , downloadBufferReadPosition(0) , downloadBufferCurrentSize(0) , downloadZerocopyBuffer(nullptr) - , pendingDownloadDataEmissions(QSharedPointer<QAtomicInt>::create()) - , pendingDownloadProgressEmissions(QSharedPointer<QAtomicInt>::create()) + , pendingDownloadDataEmissions(std::make_shared<QAtomicInt>()) + , pendingDownloadProgressEmissions(std::make_shared<QAtomicInt>()) #ifndef QT_NO_SSL , pendingIgnoreAllSslErrors(false) #endif @@ -468,19 +481,22 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = (QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); + + auto requestHeaders = request.headers(); if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { // If the request does not already specify preferred cache-control // force reload from the network and tell any caching proxy servers to reload too - if (!request.rawHeaderList().contains("Cache-Control")) { - httpRequest.setHeaderField("Cache-Control", "no-cache"); - httpRequest.setHeaderField("Pragma", "no-cache"); + if (!requestHeaders.contains(QHttpHeaders::WellKnownHeader::CacheControl)) { + const auto noCache = "no-cache"_ba; + httpRequest.setHeaderField(cacheControlName(), noCache); + httpRequest.setHeaderField("Pragma"_ba, noCache); } return false; } // The disk cache API does not currently support partial content retrieval. // That is why we don't use the disk cache for any such requests. - if (request.hasRawHeader("Range")) + if (requestHeaders.contains(QHttpHeaders::WellKnownHeader::Range)) return false; QAbstractNetworkCache *nc = managerPrivate->networkCache; @@ -494,24 +510,30 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (!metaData.saveToDisk()) return false; - QNetworkHeadersPrivate cacheHeaders; - QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); + QHttpHeaders cacheHeaders = metaData.headers(); - it = cacheHeaders.findRawHeader("etag"); - if (it != cacheHeaders.rawHeaders.constEnd()) - httpRequest.setHeaderField("If-None-Match", it->second); + const auto sizeOpt = QNetworkHeadersPrivate::toInt( + cacheHeaders.value(QHttpHeaders::WellKnownHeader::ContentLength)); + if (sizeOpt) { + std::unique_ptr<QIODevice> data(nc->data(httpRequest.url())); + if (!data || data->size() < sizeOpt.value()) + return false; // The data is smaller than the content-length specified + } + + auto value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::ETag); + if (!value.empty()) + httpRequest.setHeaderField("If-None-Match"_ba, value.toByteArray()); QDateTime lastModified = metaData.lastModified(); if (lastModified.isValid()) - httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); + httpRequest.setHeaderField("If-Modified-Since"_ba, QNetworkHeadersPrivate::toHttpDate(lastModified)); - it = cacheHeaders.findRawHeader("Cache-Control"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) + value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::CacheControl); + if (!value.empty()) { + QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(value); + if (cacheControl.contains("must-revalidate"_ba)) return false; - if (cacheControl.contains("no-cache")) + if (cacheControl.contains("no-cache"_ba)) return false; } @@ -535,16 +557,15 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h * now * is the current (local) time */ - qint64 age_value = 0; - it = cacheHeaders.findRawHeader("age"); - if (it != cacheHeaders.rawHeaders.constEnd()) - age_value = it->second.toLongLong(); + const auto ageOpt = QNetworkHeadersPrivate::toInt( + cacheHeaders.value(QHttpHeaders::WellKnownHeader::Age)); + const qint64 age_value = ageOpt.value_or(0); QDateTime dateHeader; qint64 date_value = 0; - it = cacheHeaders.findRawHeader("date"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second); + value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::Date); + if (!value.empty()) { + dateHeader = QNetworkHeadersPrivate::fromHttpDate(value); date_value = dateHeader.toSecsSinceEpoch(); } @@ -566,10 +587,11 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (lastModified.isValid() && dateHeader.isValid()) { qint64 diff = lastModified.secsTo(dateHeader); freshness_lifetime = diff / 10; - if (httpRequest.headerField("Warning").isEmpty()) { + const auto warningHeader = "Warning"_ba; + if (httpRequest.headerField(warningHeader).isEmpty()) { QDateTime dt = currentDateTime.addSecs(current_age); if (currentDateTime.daysTo(dt) > 1) - httpRequest.setHeaderField("Warning", "113"); + httpRequest.setHeaderField(warningHeader, "113"_ba); } } @@ -625,13 +647,11 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq httpRequest.setRedirectCount(newHttpRequest.maximumRedirectsAllowed()); QString scheme = url.scheme(); - bool ssl = (scheme == QLatin1String("https") - || scheme == QLatin1String("preconnect-https")); + bool ssl = (scheme == "https"_L1 || scheme == "preconnect-https"_L1); q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); httpRequest.setSsl(ssl); - bool preConnect = (scheme == QLatin1String("preconnect-http") - || scheme == QLatin1String("preconnect-https")); + bool preConnect = (scheme == "preconnect-http"_L1 || scheme == "preconnect-https"_L1); httpRequest.setPreConnect(preConnect); #ifndef QT_NO_NETWORKPROXY @@ -670,22 +690,26 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq } #endif - auto redirectPolicy = QNetworkRequest::ManualRedirectPolicy; + auto redirectPolicy = QNetworkRequest::NoLessSafeRedirectPolicy; const QVariant value = newHttpRequest.attribute(QNetworkRequest::RedirectPolicyAttribute); if (value.isValid()) redirectPolicy = qvariant_cast<QNetworkRequest::RedirectPolicy>(value); - else if (newHttpRequest.attribute(QNetworkRequest::FollowRedirectsAttribute).toBool()) - redirectPolicy = QNetworkRequest::NoLessSafeRedirectPolicy; httpRequest.setRedirectPolicy(redirectPolicy); httpRequest.setPriority(convert(newHttpRequest.priority())); + loadingFromCache = false; switch (operation) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); - if (loadFromCacheIfAllowed(httpRequest)) + // If the request has a body, createUploadByteDevice() and don't use caching + if (outgoingData) { + invalidateCache(); + createUploadByteDevice(); + } else if (loadFromCacheIfAllowed(httpRequest)) { return; // no need to send the request! :) + } break; case QNetworkAccessManager::HeadOperation: @@ -723,16 +747,16 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq break; // can't happen } - QList<QByteArray> headers = newHttpRequest.rawHeaderList(); + QHttpHeaders newRequestHeaders = newHttpRequest.headers(); if (resumeOffset != 0) { - const int rangeIndex = headers.indexOf("Range"); - if (rangeIndex != -1) { + if (newRequestHeaders.contains(QHttpHeaders::WellKnownHeader::Range)) { // Need to adjust resume offset for user specified range - headers.removeAt(rangeIndex); - // We've already verified that requestRange starts with "bytes=", see canResume. - QByteArray requestRange = newHttpRequest.rawHeader("Range").mid(6); + const auto rangeHeader = newRequestHeaders.value(QHttpHeaders::WellKnownHeader::Range); + const auto requestRange = QByteArrayView(rangeHeader).mid(bytesEqualPrefix().size()); + + newRequestHeaders.removeAll(QHttpHeaders::WellKnownHeader::Range); int index = requestRange.indexOf('-'); @@ -740,23 +764,34 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq quint64 requestEndOffset = requestRange.mid(index + 1).toULongLong(); // In case an end offset is not given it is skipped from the request range - requestRange = "bytes=" + QByteArray::number(resumeOffset + requestStartOffset) + + QByteArray newRange = bytesEqualPrefix() + QByteArray::number(resumeOffset + requestStartOffset) + '-' + (requestEndOffset ? QByteArray::number(requestEndOffset) : QByteArray()); - httpRequest.setHeaderField("Range", requestRange); + httpRequest.setHeaderField(rangeName(), newRange); } else { - httpRequest.setHeaderField("Range", "bytes=" + QByteArray::number(resumeOffset) + '-'); + httpRequest.setHeaderField(rangeName(), bytesEqualPrefix() + QByteArray::number(resumeOffset) + '-'); } } - for (const QByteArray &header : qAsConst(headers)) - httpRequest.setHeaderField(header, newHttpRequest.rawHeader(header)); + for (int i = 0; i < newRequestHeaders.size(); i++) { + const auto name = newRequestHeaders.nameAt(i); + const auto value = newRequestHeaders.valueAt(i); + httpRequest.setHeaderField(QByteArray(name.data(), name.size()), value.toByteArray()); + } if (newHttpRequest.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool()) httpRequest.setPipeliningAllowed(true); - if (request.attribute(QNetworkRequest::Http2AllowedAttribute).toBool()) - httpRequest.setHTTP2Allowed(true); + if (auto allowed = request.attribute(QNetworkRequest::Http2AllowedAttribute); + allowed.isValid() && allowed.canConvert<bool>()) { + httpRequest.setHTTP2Allowed(allowed.value<bool>()); + } + auto h2cAttribute = request.attribute(QNetworkRequest::Http2CleartextAllowedAttribute); + // ### Qt7: Stop checking the environment variable + if (h2cAttribute.toBool() + || (!h2cAttribute.isValid() && qEnvironmentVariableIsSet("QT_NETWORK_H2C_ALLOWED"))) { + httpRequest.setH2cAllowed(true); + } if (request.attribute(QNetworkRequest::Http2DirectAttribute).toBool()) { // Intentionally mutually exclusive - cannot be both direct and 'allowed' @@ -778,10 +813,24 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq QHttpThreadDelegate *delegate = new QHttpThreadDelegate; // Propagate Http/2 settings: delegate->http2Parameters = request.http2Configuration(); + delegate->http1Parameters = request.http1Configuration(); + + if (request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).isValid()) + delegate->connectionCacheExpiryTimeoutSeconds = request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).toInt(); // For the synchronous HTTP, this is the normal way the delegate gets deleted // For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished - QObject::connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater())); + QMetaObject::Connection threadFinishedConnection = + QObject::connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater())); + + // QTBUG-88063: When 'delegate' is deleted the connection will be added to 'thread''s orphaned + // connections list. This orphaned list will be cleaned up next time 'thread' emits a signal, + // unfortunately that's the finished signal. It leads to a soft-leak so we do this to disconnect + // it on deletion so that it cleans up the orphan immediately. + QObject::connect(delegate, &QObject::destroyed, delegate, [threadFinishedConnection]() { + if (bool(threadFinishedConnection)) + QObject::disconnect(threadFinishedConnection); + }); // Set the properties it needs delegate->httpRequest = httpRequest; @@ -826,14 +875,12 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq QObject::connect(delegate, SIGNAL(downloadFinished()), q, SLOT(replyFinished()), Qt::QueuedConnection); - QObject::connect(delegate, SIGNAL(downloadMetaData(QList<QPair<QByteArray,QByteArray> >, - int, QString, bool, - QSharedPointer<char>, qint64, qint64, - bool)), - q, SLOT(replyDownloadMetaData(QList<QPair<QByteArray,QByteArray> >, - int, QString, bool, - QSharedPointer<char>, qint64, qint64, bool)), - Qt::QueuedConnection); + QObject::connect(delegate, &QHttpThreadDelegate::socketStartedConnecting, + q, &QNetworkReply::socketStartedConnecting, Qt::QueuedConnection); + QObject::connect(delegate, &QHttpThreadDelegate::requestSent, + q, &QNetworkReply::requestSent, Qt::QueuedConnection); + connect(delegate, &QHttpThreadDelegate::downloadMetaData, this, + &QNetworkReplyHttpImplPrivate::replyDownloadMetaData, Qt::QueuedConnection); QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)), q, SLOT(replyDownloadProgressSlot(qint64,qint64)), Qt::QueuedConnection); @@ -844,9 +891,6 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq q, SLOT(onRedirected(QUrl,int,int)), Qt::QueuedConnection); - QObject::connect(q, SIGNAL(redirectAllowed()), q, SLOT(followRedirect()), - Qt::QueuedConnection); - #ifndef QT_NO_SSL QObject::connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), q, SLOT(replySslConfigurationChanged(QSslConfiguration)), @@ -886,14 +930,14 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq delegate->httpRequest.setUploadByteDevice(forwardUploadDevice); // If the device in the user thread claims it has more data, keep the flow to HTTP thread going - QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), + QObject::connect(uploadByteDevice.get(), SIGNAL(readyRead()), q, SLOT(uploadByteDeviceReadyReadSlot()), Qt::QueuedConnection); // From user thread to http thread: QObject::connect(q, SIGNAL(haveUploadData(qint64,QByteArray,bool,qint64)), forwardUploadDevice, SLOT(haveDataSlot(qint64,QByteArray,bool,qint64)), Qt::QueuedConnection); - QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), + QObject::connect(uploadByteDevice.get(), SIGNAL(readyRead()), forwardUploadDevice, SIGNAL(readyRead()), Qt::QueuedConnection); @@ -916,7 +960,7 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq // use the uploadByteDevice provided to us by the QNetworkReplyImpl. // The code that is in start() makes sure it is safe to use from a thread // since it only wraps a QRingBuffer - delegate->httpRequest.setUploadByteDevice(uploadByteDevice.data()); + delegate->httpRequest.setUploadByteDevice(uploadByteDevice.get()); } } @@ -933,30 +977,20 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq if (synchronous) { emit q->startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done - if (delegate->incomingErrorCode != QNetworkReply::NoError) { - replyDownloadMetaData - (delegate->incomingHeaders, - delegate->incomingStatusCode, - delegate->incomingReasonPhrase, - delegate->isPipeliningUsed, - QSharedPointer<char>(), - delegate->incomingContentLength, - delegate->removedContentLength, - delegate->isHttp2Used); - replyDownloadData(delegate->synchronousDownloadData); + replyDownloadMetaData + (delegate->incomingHeaders, + delegate->incomingStatusCode, + delegate->incomingReasonPhrase, + delegate->isPipeliningUsed, + QSharedPointer<char>(), + delegate->incomingContentLength, + delegate->removedContentLength, + delegate->isHttp2Used, + delegate->isCompressed); + replyDownloadData(delegate->synchronousDownloadData); + + if (delegate->incomingErrorCode != QNetworkReply::NoError) httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail); - } else { - replyDownloadMetaData - (delegate->incomingHeaders, - delegate->incomingStatusCode, - delegate->incomingReasonPhrase, - delegate->isPipeliningUsed, - QSharedPointer<char>(), - delegate->incomingContentLength, - delegate->removedContentLength, - delegate->isHttp2Used); - replyDownloadData(delegate->synchronousDownloadData); - } thread->quit(); thread->wait(QDeadlineTimer(5000)); @@ -1027,23 +1061,79 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) if (!q->isOpen()) return; + // cache this, we need it later and it's invalidated when dealing with compressed data + auto dataSize = d.size(); + if (cacheEnabled && isCachingAllowed() && !cacheSaveDevice) initCacheSaveDevice(); + if (decompressHelper.isValid()) { + qint64 uncompressedBefore = -1; + if (decompressHelper.isCountingBytes()) + uncompressedBefore = decompressHelper.uncompressedSize(); + + decompressHelper.feed(std::move(d)); + + if (!decompressHelper.isValid()) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(decompressHelper.errorString())); + decompressHelper.clear(); + return; + } + + if (!isHttpRedirectResponse()) { + if (decompressHelper.isCountingBytes()) + bytesDownloaded += (decompressHelper.uncompressedSize() - uncompressedBefore); + setupTransferTimeout(); + } + + if (synchronous) { + d = QByteArray(); + const qsizetype increments = 16 * 1024; + qint64 bytesRead = 0; + while (decompressHelper.hasData()) { + quint64 nextSize = quint64(d.size()) + quint64(increments); + if (nextSize > quint64(std::numeric_limits<QByteArray::size_type>::max())) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", + "Data downloaded is too large to store")); + decompressHelper.clear(); + return; + } + d.resize(nextSize); + bytesRead += decompressHelper.read(d.data() + bytesRead, increments); + if (!decompressHelper.isValid()) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(decompressHelper.errorString())); + decompressHelper.clear(); + return; + } + } + d.resize(bytesRead); + // we're synchronous so we're not calling this function again; reset the decompressHelper + decompressHelper.clear(); + } + } + // This is going to look a little strange. When downloading data while a // HTTP redirect is happening (and enabled), we write the redirect // response to the cache. However, we do not append it to our internal // buffer as that will contain the response data only for the final // response - if (cacheSaveDevice) + // Note: For compressed data this is done in readData() + if (cacheSaveDevice && !decompressHelper.isValid()) { cacheSaveDevice->write(d); + } - if (!isHttpRedirectResponse()) { + // if decompressHelper is valid then we have compressed data, and this is handled above + if (!decompressHelper.isValid() && !isHttpRedirectResponse()) { buffer.append(d); - bytesDownloaded += d.size(); + bytesDownloaded += dataSize; setupTransferTimeout(); } - bytesBuffered += d.size(); + bytesBuffered += dataSize; int pendingSignals = pendingDownloadDataEmissions->fetchAndSubAcquire(1) - 1; if (pendingSignals > 0) { @@ -1057,17 +1147,26 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) if (isHttpRedirectResponse()) return; - QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + // This can occur when downloading compressed data as some of the data may be the content + // encoding's header. Don't emit anything for this. + if (lastReadyReadEmittedSize == bytesDownloaded) { + if (readBufferMaxSize) + emit q->readBufferFreed(dataSize); + return; + } + lastReadyReadEmittedSize = bytesDownloaded; + + const auto totalSizeOpt = QNetworkHeadersPrivate::toInt( + headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); emit q->readyRead(); - // emit readyRead before downloadProgress incase this will cause events to be + // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). - if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) { + if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval + && (!decompressHelper.isValid() || decompressHelper.isCountingBytes())) { downloadProgressSignalChoke.restart(); - emit q->downloadProgress(bytesDownloaded, - totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + emit q->downloadProgress(bytesDownloaded, totalSizeOpt.value_or(-1)); } - } void QNetworkReplyHttpImplPrivate::replyFinished() @@ -1138,13 +1237,12 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt // equal to "80", the port component value MUST be preserved; // otherwise, if the URI does not contain an explicit port // component, the UA MUST NOT add one. - url.setScheme(QLatin1String("https")); + url.setScheme("https"_L1); if (url.port() == 80) url.setPort(443); } - const bool isLessSafe = schemeBefore == QLatin1String("https") - && url.scheme() == QLatin1String("http"); + const bool isLessSafe = schemeBefore == "https"_L1 && url.scheme() == "http"_L1; if (httpRequest.redirectPolicy() == QNetworkRequest::NoLessSafeRedirectPolicy && isLessSafe) { error(QNetworkReply::InsecureRedirectError, @@ -1152,13 +1250,19 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt return; } + // If the original operation was a GET with a body and the status code is either + // 307 or 308 then keep the message body + const bool getOperationKeepsBody = (operation == QNetworkAccessManager::GetOperation) + && (httpStatus == 307 || httpStatus == 308); + redirectRequest = createRedirectRequest(originalRequest, url, maxRedirectsRemaining); operation = getRedirectOperation(operation, httpStatus); // Clear stale headers, the relevant ones get set again later httpRequest.clearHeaders(); - if (operation == QNetworkAccessManager::GetOperation - || operation == QNetworkAccessManager::HeadOperation) { + auto newHeaders = redirectRequest.headers(); + if ((operation == QNetworkAccessManager::GetOperation + || operation == QNetworkAccessManager::HeadOperation) && !getOperationKeepsBody) { // possibly changed from not-GET/HEAD to GET/HEAD, make sure to get rid of upload device uploadByteDevice.reset(); uploadByteDevicePosition = 0; @@ -1171,18 +1275,20 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt outgoingData = nullptr; outgoingDataBuffer.reset(); // We need to explicitly unset these headers so they're not reapplied to the httpRequest - redirectRequest.setHeader(QNetworkRequest::ContentLengthHeader, QVariant()); - redirectRequest.setHeader(QNetworkRequest::ContentTypeHeader, QVariant()); + newHeaders.removeAll(QHttpHeaders::WellKnownHeader::ContentLength); + newHeaders.removeAll(QHttpHeaders::WellKnownHeader::ContentType); } if (const QNetworkCookieJar *const cookieJar = manager->cookieJar()) { auto cookies = cookieJar->cookiesForUrl(url); if (!cookies.empty()) { - redirectRequest.setHeader(QNetworkRequest::KnownHeaders::CookieHeader, - QVariant::fromValue(cookies)); + auto cookieHeader = QNetworkHeadersPrivate::fromCookieList(cookies); + newHeaders.replaceOrAppend(QHttpHeaders::WellKnownHeader::Cookie, cookieHeader); } } + redirectRequest.setHeaders(std::move(newHeaders)); + if (httpRequest.redirectPolicy() != QNetworkRequest::UserVerifiedRedirectPolicy) followRedirect(); @@ -1194,8 +1300,8 @@ void QNetworkReplyHttpImplPrivate::followRedirect() Q_Q(QNetworkReplyHttpImpl); Q_ASSERT(managerPrivate); - rawHeaders.clear(); - cookedHeaders.clear(); + decompressHelper.clear(); + clearHeaders(); if (managerPrivate->thread) managerPrivate->thread->disconnect(); @@ -1204,6 +1310,8 @@ void QNetworkReplyHttpImplPrivate::followRedirect() q, [this]() { postRequest(redirectRequest); }, Qt::QueuedConnection); } +static constexpr QLatin1StringView locationHeader() noexcept { return "location"_L1; } + void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) { Q_Q(QNetworkReplyHttpImpl); @@ -1216,20 +1324,20 @@ void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) // What do we do about the caching of the HTML note? // The response to a 303 MUST NOT be cached, while the response to // all of the others is cacheable if the headers indicate it to be - QByteArray header = q->rawHeader("location"); + QByteArrayView header = q->headers().value(locationHeader()); QUrl url = QUrl(QString::fromUtf8(header)); if (!url.isValid()) - url = QUrl(QLatin1String(header)); + url = QUrl(QLatin1StringView(header)); q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url); } } -void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByteArray,QByteArray> > &hm, +void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QHttpHeaders &hm, int sc, const QString &rp, bool pu, QSharedPointer<char> db, qint64 contentLength, qint64 removedContentLength, - bool h2Used) + bool h2Used, bool isCompressed) { Q_Q(QNetworkReplyHttpImpl); Q_UNUSED(contentLength); @@ -1243,7 +1351,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte // RFC6797, 8.1 // If an HTTP response is received over insecure transport, the UA MUST // ignore any present STS header field(s). - if (url.scheme() == QLatin1String("https") && managerPrivate->stsEnabled) + if (url.scheme() == "https"_L1 && managerPrivate->stsEnabled) managerPrivate->stsCache.updateFromHeaders(hm, url); #endif // Download buffer @@ -1257,30 +1365,40 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte q->setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); q->setAttribute(QNetworkRequest::Http2WasUsedAttribute, h2Used); + // A user having manually defined which encodings they accept is, for + // somwehat unknown (presumed legacy compatibility) reasons treated as + // disabling our decompression: + const bool autoDecompress = !request.headers().contains(QHttpHeaders::WellKnownHeader::AcceptEncoding); + const bool shouldDecompress = isCompressed && autoDecompress; // reconstruct the HTTP header - QList<QPair<QByteArray, QByteArray> > headerMap = hm; - QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(), - end = headerMap.constEnd(); - for (; it != end; ++it) { - QByteArray value = q->rawHeader(it->first); + auto h = q->headers(); + for (qsizetype i = 0; i < hm.size(); ++i) { + const auto key = hm.nameAt(i); + const auto originValue = hm.valueAt(i); // Reset any previous "location" header set in the reply. In case of // redirects, we don't want to 'append' multiple location header values, // rather we keep only the latest one - if (it->first.toLower() == "location") - value.clear(); - - if (!value.isEmpty()) { - // Why are we appending values for headers which are already - // present? - if (it->first.compare("set-cookie", Qt::CaseInsensitive) == 0) - value += '\n'; - else - value += ", "; + if (key.compare(locationHeader(), Qt::CaseInsensitive) == 0) + h.removeAll(key); + + if (shouldDecompress && !decompressHelper.isValid() && key == "content-encoding"_L1) { + if (!synchronous) // with synchronous all the data is expected to be handled at once + decompressHelper.setCountingBytesEnabled(true); + + if (!decompressHelper.setEncoding(originValue)) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Failed to initialize decompression: %1") + .arg(decompressHelper.errorString())); + return; + } + decompressHelper.setDecompressedSafetyCheckThreshold( + request.decompressedSafetyCheckThreshold()); } - value += it->second; - q->setRawHeader(it->first, value); + + h.append(key, originValue); } + q->setHeaders(std::move(h)); q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); @@ -1295,14 +1413,11 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte QAbstractNetworkCache *nc = managerPrivate->networkCache; if (nc) { QNetworkCacheMetaData metaData = nc->metaData(httpRequest.url()); - QNetworkHeadersPrivate cacheHeaders; - cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); - QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - it = cacheHeaders.findRawHeader("Cache-Control"); + auto value = metaData.headers().value(QHttpHeaders::WellKnownHeader::CacheControl); bool mustReValidate = false; - if (it != cacheHeaders.rawHeaders.constEnd()) { - QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) + if (!value.empty()) { + QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(value); + if (cacheControl.contains("must-revalidate"_ba)) mustReValidate = true; } if (!mustReValidate && sendCacheContents(metaData)) @@ -1372,7 +1487,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadProgressSlot(qint64 bytesReceive downloadBufferCurrentSize = bytesReceived; // Only emit readyRead when actual data is there - // emit readyRead before downloadProgress incase this will cause events to be + // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). if (bytesDownloaded > 0) emit q->readyRead(); @@ -1540,16 +1655,21 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); q->setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); - QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders(); - QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), - end = rawHeaders.constEnd(); + QHttpHeaders cachedHeaders = metaData.headers(); + QHttpHeaders h = headers(); QUrl redirectUrl; - for ( ; it != end; ++it) { - if (httpRequest.isFollowRedirects() && - !it->first.compare("location", Qt::CaseInsensitive)) - redirectUrl = QUrl::fromEncoded(it->second); - setRawHeader(it->first, it->second); + for (qsizetype i = 0; i < cachedHeaders.size(); ++i) { + const auto name = cachedHeaders.nameAt(i); + const auto value = cachedHeaders.valueAt(i); + + if (httpRequest.isFollowRedirects() + && !name.compare(locationHeader(), Qt::CaseInsensitive)) { + redirectUrl = QUrl::fromEncoded(value); + } + + h.replaceOrAppend(name, value); } + setHeaders(std::move(h)); if (!isHttpRedirectResponse()) checkForRedirect(status); @@ -1583,33 +1703,43 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData return true; } +static auto caseInsensitiveCompare(QByteArrayView value) +{ + return [value](QByteArrayView element) + { + return value.compare(element, Qt::CaseInsensitive) == 0; + }; +} + +static bool isHopByHop(QByteArrayView header) +{ + constexpr QByteArrayView headers[] = { "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade"}; + return std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(header)); +} + QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const { Q_Q(const QNetworkReplyHttpImpl); QNetworkCacheMetaData metaData = oldMetaData; + QHttpHeaders cacheHeaders = metaData.headers(); + + const auto newHeaders = q->headers(); + for (qsizetype i = 0; i < newHeaders.size(); ++i) { + const auto name = newHeaders.nameAt(i); + const auto value = newHeaders.valueAt(i); - QNetworkHeadersPrivate cacheHeaders; - cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); - QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - - const QList<QByteArray> newHeaders = q->rawHeaderList(); - for (QByteArray header : newHeaders) { - QByteArray originalHeader = header; - header = header.toLower(); - bool hop_by_hop = - (header == "connection" - || header == "keep-alive" - || header == "proxy-authenticate" - || header == "proxy-authorization" - || header == "te" - || header == "trailers" - || header == "transfer-encoding" - || header == "upgrade"); - if (hop_by_hop) + if (isHopByHop(name)) continue; - if (header == "set-cookie") + if (name.compare("set-cookie", Qt::CaseInsensitive) == 0) continue; // for 4.6.0, we were planning to not store the date header in the @@ -1622,51 +1752,47 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe //continue; // Don't store Warning 1xx headers - if (header == "warning") { - QByteArray v = q->rawHeader(header); - if (v.length() == 3 - && v[0] == '1' - && v[1] >= '0' && v[1] <= '9' - && v[2] >= '0' && v[2] <= '9') + if (name.compare("warning", Qt::CaseInsensitive) == 0) { + if (value.size() == 3 + && value[0] == '1' + && isAsciiDigit(value[1]) + && isAsciiDigit(value[2])) continue; } - it = cacheHeaders.findRawHeader(header); - if (it != cacheHeaders.rawHeaders.constEnd()) { + if (cacheHeaders.contains(name)) { // Match the behavior of Firefox and assume Cache-Control: "no-transform" - if (header == "content-encoding" - || header == "content-range" - || header == "content-type") + constexpr QByteArrayView headers[]= + {"content-encoding", "content-range", "content-type"}; + if (std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(name))) continue; } // IIS has been known to send "Content-Length: 0" on 304 responses, so // ignore this too - if (header == "content-length" && statusCode == 304) + if (statusCode == 304 && name.compare("content-length", Qt::CaseInsensitive) == 0) continue; #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) - QByteArray n = q->rawHeader(header); - QByteArray o; - if (it != cacheHeaders.rawHeaders.constEnd()) - o = (*it).second; - if (n != o && header != "date") { - qDebug() << "replacing" << header; + QByteArrayView n = newHeaders.value(name); + QByteArrayView o = cacheHeaders.value(name); + if (n != o && name.compare("date", Qt::CaseInsensitive) != 0) { + qDebug() << "replacing" << name; qDebug() << "new" << n; qDebug() << "old" << o; } #endif - cacheHeaders.setRawHeader(originalHeader, q->rawHeader(header)); + cacheHeaders.replaceOrAppend(name, value); } - metaData.setRawHeaders(cacheHeaders.rawHeaders); + metaData.setHeaders(cacheHeaders); bool checkExpired = true; QHash<QByteArray, QByteArray> cacheControl; - it = cacheHeaders.findRawHeader("Cache-Control"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - cacheControl = parseHttpOptionHeader(it->second); - QByteArray maxAge = cacheControl.value("max-age"); + auto value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::CacheControl); + if (!value.empty()) { + cacheControl = parseHttpOptionHeader(value); + QByteArray maxAge = cacheControl.value("max-age"_ba); if (!maxAge.isEmpty()) { checkExpired = false; QDateTime dt = QDateTime::currentDateTimeUtc(); @@ -1675,16 +1801,18 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe } } if (checkExpired) { - it = cacheHeaders.findRawHeader("expires"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(it->second); + if (const auto value = cacheHeaders.value( + QHttpHeaders::WellKnownHeader::Expires); !value.isEmpty()) { + QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(value); metaData.setExpirationDate(expiredDateTime); } } - it = cacheHeaders.findRawHeader("last-modified"); - if (it != cacheHeaders.rawHeaders.constEnd()) - metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(it->second)); + if (const auto value = cacheHeaders.value( + QHttpHeaders::WellKnownHeader::LastModified); !value.isEmpty()) { + metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(value)); + } + bool canDiskCache; // only cache GET replies by default, all other replies (POST, PUT, DELETE) @@ -1693,7 +1821,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe canDiskCache = true; // HTTP/1.1. Check the Cache-Control header - if (cacheControl.contains("no-store")) + if (cacheControl.contains("no-store"_ba)) canDiskCache = false; // responses to POST might be cacheable @@ -1702,7 +1830,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe canDiskCache = false; // some pages contain "expires:" and "cache-control: no-cache" field, // so we only might cache POST requests if we get "cache-control: max-age ..." - if (cacheControl.contains("max-age")) + if (cacheControl.contains("max-age"_ba)) canDiskCache = true; // responses to PUT and DELETE are not cacheable @@ -1732,15 +1860,17 @@ bool QNetworkReplyHttpImplPrivate::canResume() const if (operation != QNetworkAccessManager::GetOperation) return false; + const auto h = q->headers(); + // Can only resume if server/resource supports Range header. - QByteArray acceptRangesheaderName("Accept-Ranges"); - if (!q->hasRawHeader(acceptRangesheaderName) || q->rawHeader(acceptRangesheaderName) == "none") + const auto acceptRanges = h.value(QHttpHeaders::WellKnownHeader::AcceptRanges); + if (acceptRanges.empty() || acceptRanges == "none") return false; // We only support resuming for byte ranges. - if (request.hasRawHeader("Range")) { - QByteArray range = request.rawHeader("Range"); - if (!range.startsWith("bytes=")) + const auto range = h.value(QHttpHeaders::WellKnownHeader::Range); + if (!range.empty()) { + if (!range.startsWith(bytesEqualPrefix())) return false; } @@ -1759,7 +1889,9 @@ void QNetworkReplyHttpImplPrivate::setResumeOffset(quint64 offset) void QNetworkReplyHttpImplPrivate::_q_startOperation() { - if (state == Working) // ensure this function is only being called once + // Ensure this function is only being called once, and not at all if we were + // cancelled + if (state >= Working) return; state = Working; @@ -1786,10 +1918,10 @@ void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead() // Needs to be done where sendCacheContents() (?) of HTTP is emitting // metaDataChanged ? + const auto totalSizeOpt = QNetworkHeadersPrivate::toInt( + headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); - QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); - - // emit readyRead before downloadProgress incase this will cause events to be + // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). if (!(isHttpRedirectResponse())) { @@ -1798,8 +1930,7 @@ void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead() if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) { downloadProgressSignalChoke.restart(); - emit q->downloadProgress(bytesDownloaded, - totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + emit q->downloadProgress(bytesDownloaded, totalSizeOpt.value_or(-1)); } } @@ -1866,7 +1997,7 @@ void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingData() if (!outgoingDataBuffer) { // first call, create our buffer - outgoingDataBuffer = QSharedPointer<QRingBuffer>::create(); + outgoingDataBuffer = std::make_shared<QRingBuffer>(); QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); @@ -1919,9 +2050,9 @@ void QNetworkReplyHttpImplPrivate::setupTransferTimeout() Qt::QueuedConnection); } transferTimeout->stop(); - if (request.transferTimeout()) { + if (request.transferTimeoutAsDuration() > 0ms) { transferTimeout->setSingleShot(true); - transferTimeout->setInterval(request.transferTimeout()); + transferTimeout->setInterval(request.transferTimeoutAsDuration()); QMetaObject::invokeMethod(transferTimeout, "start", Qt::QueuedConnection); @@ -1966,10 +2097,10 @@ QNonContiguousByteDevice* QNetworkReplyHttpImplPrivate::createUploadByteDevice() // We want signal emissions only for normal asynchronous uploads if (!synchronous) - QObject::connect(uploadByteDevice.data(), SIGNAL(readProgress(qint64,qint64)), + QObject::connect(uploadByteDevice.get(), SIGNAL(readProgress(qint64,qint64)), q, SLOT(emitReplyUploadProgress(qint64,qint64))); - return uploadByteDevice.data(); + return uploadByteDevice.get(); } void QNetworkReplyHttpImplPrivate::_q_finished() @@ -1986,11 +2117,16 @@ void QNetworkReplyHttpImplPrivate::finished() if (state == Finished || state == Aborted) return; - QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + const auto totalSizeOpt = QNetworkHeadersPrivate::toInt( + headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); + const qint64 totalSize = totalSizeOpt.value_or(-1); - // if we don't know the total size of or we received everything save the cache - if (totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize) + // if we don't know the total size of or we received everything save the cache. + // If the data is compressed then this is done in readData() + if ((totalSize == -1 || bytesDownloaded == totalSize) + && !decompressHelper.isValid()) { completeCacheSave(); + } // We check for errorCode too as in case of SSL handshake failure, we still // get the HTTP redirect status code (301, 303 etc) @@ -2000,10 +2136,10 @@ void QNetworkReplyHttpImplPrivate::finished() state = Finished; q->setFinished(true); - if (totalSize.isNull() || totalSize == -1) { + if (totalSize == -1) { emit q->downloadProgress(bytesDownloaded, bytesDownloaded); } else { - emit q->downloadProgress(bytesDownloaded, totalSize.toLongLong()); + emit q->downloadProgress(bytesDownloaded, totalSize); } if (bytesUploaded == -1 && (outgoingData || outgoingDataBuffer)) @@ -2024,7 +2160,9 @@ void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, c Q_Q(QNetworkReplyHttpImpl); // Can't set and emit multiple errors. if (errorCode != QNetworkReply::NoError) { - qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."); + // But somewhat unavoidable if we have cancelled the request: + if (errorCode != QNetworkReply::OperationCanceledError) + qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."); return; } @@ -2045,14 +2183,15 @@ void QNetworkReplyHttpImplPrivate::_q_metaDataChanged() // 1. do we have cookies? // 2. are we allowed to set them? Q_ASSERT(manager); - const auto it = cookedHeaders.constFind(QNetworkRequest::SetCookieHeader); - if (it != cookedHeaders.cend() + + const auto cookiesOpt = QNetworkHeadersPrivate::toSetCookieList( + headers().values(QHttpHeaders::WellKnownHeader::SetCookie)); + const auto cookies = cookiesOpt.value_or(QList<QNetworkCookie>()); + if (!cookies.empty() && request.attribute(QNetworkRequest::CookieSaveControlAttribute, QNetworkRequest::Automatic).toInt() == QNetworkRequest::Automatic) { QNetworkCookieJar *jar = manager->cookieJar(); if (jar) { - QList<QNetworkCookie> cookies = - qvariant_cast<QList<QNetworkCookie> >(it.value()); jar->setCookiesFromUrl(cookies, url); } } @@ -2116,3 +2255,5 @@ void QNetworkReplyHttpImplPrivate::completeCacheSave() } QT_END_NAMESPACE + +#include "moc_qnetworkreplyhttpimpl_p.cpp" |