diff options
Diffstat (limited to 'src/network/access/qnetworkreplyhttpimpl.cpp')
-rw-r--r-- | src/network/access/qnetworkreplyhttpimpl.cpp | 520 |
1 files changed, 274 insertions, 246 deletions
diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 502692084a..3e1fe761ee 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtNetwork module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only //#define QNETWORKACCESSHTTPBACKEND_DEBUG @@ -57,6 +21,7 @@ #include "QtCore/qcoreapplication.h" #include <QtCore/private/qthread_p.h> +#include <QtCore/private/qtools_p.h> #include "qnetworkcookiejar.h" #include "qnetconmonitor_p.h" @@ -67,16 +32,18 @@ QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; +using namespace QtMiscUtils; +using namespace std::chrono_literals; + class QNetworkProxy; -static inline bool isSeparator(char c) -{ - static const char separators[] = "()<>@,;:\\\"/[]?={}"; - return isLWS(c) || strchr(separators, c) != nullptr; -} +static inline QByteArray rangeName() { return "Range"_ba; } +static inline QByteArray cacheControlName() { return "Cache-Control"_ba; } +static constexpr QByteArrayView bytesEqualPrefix() noexcept { return "bytes="; } // ### merge with nextField in cookiejar.cpp -static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &header) +static QHash<QByteArray, QByteArray> parseHttpOptionHeader(QByteArrayView header) { // The HTTP header is of the form: // header = #1(directives) @@ -88,7 +55,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea while (true) { // skip spaces pos = nextNonWhitespace(header, pos); - if (pos == header.length()) + if (pos == header.size()) return result; // end of parsing // pos points to a non-whitespace @@ -102,36 +69,36 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea // of the header, whichever comes first int end = comma; if (end == -1) - end = header.length(); + end = header.size(); if (equal != -1 && end > equal) end = equal; // equal sign comes before comma/end - QByteArray key = QByteArray(header.constData() + pos, end - pos).trimmed().toLower(); + const auto key = header.sliced(pos, end - pos).trimmed(); pos = end + 1; if (uint(equal) < uint(comma)) { // case: token "=" (token | quoted-string) // skip spaces pos = nextNonWhitespace(header, pos); - if (pos == header.length()) + if (pos == header.size()) // huh? Broken header return result; QByteArray value; - value.reserve(header.length() - pos); + value.reserve(header.size() - pos); if (header.at(pos) == '"') { // case: quoted-string // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) // qdtext = <any TEXT except <">> // quoted-pair = "\" CHAR ++pos; - while (pos < header.length()) { + while (pos < header.size()) { char c = header.at(pos); if (c == '"') { // end of quoted text break; } else if (c == '\\') { ++pos; - if (pos >= header.length()) + if (pos >= header.size()) // broken header return result; c = header.at(pos); @@ -141,8 +108,13 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea ++pos; } } else { + const auto isSeparator = [](char c) { + static const char separators[] = "()<>@,;:\\\"/[]?={}"; + return isLWS(c) || strchr(separators, c) != nullptr; + }; + // case: token - while (pos < header.length()) { + while (pos < header.size()) { char c = header.at(pos); if (isSeparator(c)) break; @@ -151,7 +123,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea } } - result.insert(key, value); + result.insert(key.toByteArray().toLower(), value); // find the comma now: comma = header.indexOf(',', pos); @@ -161,7 +133,7 @@ static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &hea } else { // case: token // key is already set - result.insert(key, QByteArray()); + result.insert(key.toByteArray().toLower(), QByteArray()); } } } @@ -182,10 +154,13 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage d->outgoingData = outgoingData; d->url = request.url(); #ifndef QT_NO_SSL - if (request.url().scheme() == QLatin1String("https")) + if (request.url().scheme() == "https"_L1) d->sslConfiguration.reset(new QSslConfiguration(request.sslConfiguration())); #endif + QObjectPrivate::connect(this, &QNetworkReplyHttpImpl::redirectAllowed, d, + &QNetworkReplyHttpImplPrivate::followRedirect, Qt::QueuedConnection); + // FIXME Later maybe set to Unbuffered, especially if it is zerocopy or from cache? QIODevice::open(QIODevice::ReadOnly); @@ -227,7 +202,10 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage if (bufferingDisallowed) { // if a valid content-length header for the request was supplied, we can disable buffering // if not, we will buffer anyway - if (request.header(QNetworkRequest::ContentLengthHeader).isValid()) { + + const auto sizeOpt = QNetworkHeadersPrivate::toInt( + request.headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); + if (sizeOpt) { QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection); // FIXME make direct call? } else { @@ -357,9 +335,11 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) return 0; const qint64 bytesRead = d->decompressHelper.read(data, maxlen); if (!d->decompressHelper.isValid()) { - // error occurred, error copied from QHttpNetworkConnectionPrivate::errorDetail - d->error(QNetworkReplyImpl::NetworkError::ProtocolFailure, - QCoreApplication::translate("QHttp", "Data corrupted")); + d->error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(d->decompressHelper.errorString())); + d->decompressHelper.clear(); + return -1; } if (d->cacheSaveDevice) { // Need to write to the cache now that we have the data @@ -501,19 +481,22 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = (QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); + + auto requestHeaders = request.headers(); if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { // If the request does not already specify preferred cache-control // force reload from the network and tell any caching proxy servers to reload too - if (!request.rawHeaderList().contains("Cache-Control")) { - httpRequest.setHeaderField("Cache-Control", "no-cache"); - httpRequest.setHeaderField("Pragma", "no-cache"); + if (!requestHeaders.contains(QHttpHeaders::WellKnownHeader::CacheControl)) { + const auto noCache = "no-cache"_ba; + httpRequest.setHeaderField(cacheControlName(), noCache); + httpRequest.setHeaderField("Pragma"_ba, noCache); } return false; } // The disk cache API does not currently support partial content retrieval. // That is why we don't use the disk cache for any such requests. - if (request.hasRawHeader("Range")) + if (requestHeaders.contains(QHttpHeaders::WellKnownHeader::Range)) return false; QAbstractNetworkCache *nc = managerPrivate->networkCache; @@ -527,24 +510,30 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (!metaData.saveToDisk()) return false; - QNetworkHeadersPrivate cacheHeaders; - QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); + QHttpHeaders cacheHeaders = metaData.headers(); + + const auto sizeOpt = QNetworkHeadersPrivate::toInt( + cacheHeaders.value(QHttpHeaders::WellKnownHeader::ContentLength)); + if (sizeOpt) { + std::unique_ptr<QIODevice> data(nc->data(httpRequest.url())); + if (!data || data->size() < sizeOpt.value()) + return false; // The data is smaller than the content-length specified + } - it = cacheHeaders.findRawHeader("etag"); - if (it != cacheHeaders.rawHeaders.constEnd()) - httpRequest.setHeaderField("If-None-Match", it->second); + auto value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::ETag); + if (!value.empty()) + httpRequest.setHeaderField("If-None-Match"_ba, value.toByteArray()); QDateTime lastModified = metaData.lastModified(); if (lastModified.isValid()) - httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); + httpRequest.setHeaderField("If-Modified-Since"_ba, QNetworkHeadersPrivate::toHttpDate(lastModified)); - it = cacheHeaders.findRawHeader("Cache-Control"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) + value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::CacheControl); + if (!value.empty()) { + QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(value); + if (cacheControl.contains("must-revalidate"_ba)) return false; - if (cacheControl.contains("no-cache")) + if (cacheControl.contains("no-cache"_ba)) return false; } @@ -568,16 +557,15 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h * now * is the current (local) time */ - qint64 age_value = 0; - it = cacheHeaders.findRawHeader("age"); - if (it != cacheHeaders.rawHeaders.constEnd()) - age_value = it->second.toLongLong(); + const auto ageOpt = QNetworkHeadersPrivate::toInt( + cacheHeaders.value(QHttpHeaders::WellKnownHeader::Age)); + const qint64 age_value = ageOpt.value_or(0); QDateTime dateHeader; qint64 date_value = 0; - it = cacheHeaders.findRawHeader("date"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second); + value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::Date); + if (!value.empty()) { + dateHeader = QNetworkHeadersPrivate::fromHttpDate(value); date_value = dateHeader.toSecsSinceEpoch(); } @@ -599,10 +587,11 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (lastModified.isValid() && dateHeader.isValid()) { qint64 diff = lastModified.secsTo(dateHeader); freshness_lifetime = diff / 10; - if (httpRequest.headerField("Warning").isEmpty()) { + const auto warningHeader = "Warning"_ba; + if (httpRequest.headerField(warningHeader).isEmpty()) { QDateTime dt = currentDateTime.addSecs(current_age); if (currentDateTime.daysTo(dt) > 1) - httpRequest.setHeaderField("Warning", "113"); + httpRequest.setHeaderField(warningHeader, "113"_ba); } } @@ -658,13 +647,11 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq httpRequest.setRedirectCount(newHttpRequest.maximumRedirectsAllowed()); QString scheme = url.scheme(); - bool ssl = (scheme == QLatin1String("https") - || scheme == QLatin1String("preconnect-https")); + bool ssl = (scheme == "https"_L1 || scheme == "preconnect-https"_L1); q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); httpRequest.setSsl(ssl); - bool preConnect = (scheme == QLatin1String("preconnect-http") - || scheme == QLatin1String("preconnect-https")); + bool preConnect = (scheme == "preconnect-http"_L1 || scheme == "preconnect-https"_L1); httpRequest.setPreConnect(preConnect); #ifndef QT_NO_NETWORKPROXY @@ -711,12 +698,18 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq httpRequest.setRedirectPolicy(redirectPolicy); httpRequest.setPriority(convert(newHttpRequest.priority())); + loadingFromCache = false; switch (operation) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); - if (loadFromCacheIfAllowed(httpRequest)) + // If the request has a body, createUploadByteDevice() and don't use caching + if (outgoingData) { + invalidateCache(); + createUploadByteDevice(); + } else if (loadFromCacheIfAllowed(httpRequest)) { return; // no need to send the request! :) + } break; case QNetworkAccessManager::HeadOperation: @@ -754,16 +747,16 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq break; // can't happen } - QList<QByteArray> headers = newHttpRequest.rawHeaderList(); + QHttpHeaders newRequestHeaders = newHttpRequest.headers(); if (resumeOffset != 0) { - const int rangeIndex = headers.indexOf("Range"); - if (rangeIndex != -1) { + if (newRequestHeaders.contains(QHttpHeaders::WellKnownHeader::Range)) { // Need to adjust resume offset for user specified range - headers.removeAt(rangeIndex); - // We've already verified that requestRange starts with "bytes=", see canResume. - QByteArray requestRange = newHttpRequest.rawHeader("Range").mid(6); + const auto rangeHeader = newRequestHeaders.value(QHttpHeaders::WellKnownHeader::Range); + const auto requestRange = QByteArrayView(rangeHeader).mid(bytesEqualPrefix().size()); + + newRequestHeaders.removeAll(QHttpHeaders::WellKnownHeader::Range); int index = requestRange.indexOf('-'); @@ -771,17 +764,20 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq quint64 requestEndOffset = requestRange.mid(index + 1).toULongLong(); // In case an end offset is not given it is skipped from the request range - requestRange = "bytes=" + QByteArray::number(resumeOffset + requestStartOffset) + + QByteArray newRange = bytesEqualPrefix() + QByteArray::number(resumeOffset + requestStartOffset) + '-' + (requestEndOffset ? QByteArray::number(requestEndOffset) : QByteArray()); - httpRequest.setHeaderField("Range", requestRange); + httpRequest.setHeaderField(rangeName(), newRange); } else { - httpRequest.setHeaderField("Range", "bytes=" + QByteArray::number(resumeOffset) + '-'); + httpRequest.setHeaderField(rangeName(), bytesEqualPrefix() + QByteArray::number(resumeOffset) + '-'); } } - for (const QByteArray &header : qAsConst(headers)) - httpRequest.setHeaderField(header, newHttpRequest.rawHeader(header)); + for (int i = 0; i < newRequestHeaders.size(); i++) { + const auto name = newRequestHeaders.nameAt(i); + const auto value = newRequestHeaders.valueAt(i); + httpRequest.setHeaderField(QByteArray(name.data(), name.size()), value.toByteArray()); + } if (newHttpRequest.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool()) httpRequest.setPipeliningAllowed(true); @@ -790,6 +786,12 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq allowed.isValid() && allowed.canConvert<bool>()) { httpRequest.setHTTP2Allowed(allowed.value<bool>()); } + auto h2cAttribute = request.attribute(QNetworkRequest::Http2CleartextAllowedAttribute); + // ### Qt7: Stop checking the environment variable + if (h2cAttribute.toBool() + || (!h2cAttribute.isValid() && qEnvironmentVariableIsSet("QT_NETWORK_H2C_ALLOWED"))) { + httpRequest.setH2cAllowed(true); + } if (request.attribute(QNetworkRequest::Http2DirectAttribute).toBool()) { // Intentionally mutually exclusive - cannot be both direct and 'allowed' @@ -811,6 +813,7 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq QHttpThreadDelegate *delegate = new QHttpThreadDelegate; // Propagate Http/2 settings: delegate->http2Parameters = request.http2Configuration(); + delegate->http1Parameters = request.http1Configuration(); if (request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).isValid()) delegate->connectionCacheExpiryTimeoutSeconds = request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).toInt(); @@ -872,8 +875,8 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq QObject::connect(delegate, SIGNAL(downloadFinished()), q, SLOT(replyFinished()), Qt::QueuedConnection); - QObject::connect(delegate, &QHttpThreadDelegate::socketConnecting, - q, &QNetworkReply::socketConnecting, Qt::QueuedConnection); + QObject::connect(delegate, &QHttpThreadDelegate::socketStartedConnecting, + q, &QNetworkReply::socketStartedConnecting, Qt::QueuedConnection); QObject::connect(delegate, &QHttpThreadDelegate::requestSent, q, &QNetworkReply::requestSent, Qt::QueuedConnection); connect(delegate, &QHttpThreadDelegate::downloadMetaData, this, @@ -888,9 +891,6 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq q, SLOT(onRedirected(QUrl,int,int)), Qt::QueuedConnection); - QObject::connect(q, SIGNAL(redirectAllowed()), q, SLOT(followRedirect()), - Qt::QueuedConnection); - #ifndef QT_NO_SSL QObject::connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), q, SLOT(replySslConfigurationChanged(QSslConfiguration)), @@ -1063,9 +1063,6 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) // cache this, we need it later and it's invalidated when dealing with compressed data auto dataSize = d.size(); - // Grab this to compare later (only relevant for compressed data) in case none of the data - // will be propagated to the user - const qint64 previousBytesDownloaded = bytesDownloaded; if (cacheEnabled && isCachingAllowed() && !cacheSaveDevice) initCacheSaveDevice(); @@ -1078,9 +1075,10 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) decompressHelper.feed(std::move(d)); if (!decompressHelper.isValid()) { - // error occurred, error copied from QHttpNetworkConnectionPrivate::errorDetail - error(QNetworkReplyImpl::NetworkError::ProtocolFailure, - QCoreApplication::translate("QHttp", "Data corrupted")); + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(decompressHelper.errorString())); + decompressHelper.clear(); return; } @@ -1097,12 +1095,21 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) while (decompressHelper.hasData()) { quint64 nextSize = quint64(d.size()) + quint64(increments); if (nextSize > quint64(std::numeric_limits<QByteArray::size_type>::max())) { - error(QNetworkReplyImpl::NetworkError::ProtocolFailure, - QCoreApplication::translate("QHttp", "Data corrupted")); + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", + "Data downloaded is too large to store")); + decompressHelper.clear(); return; } d.resize(nextSize); bytesRead += decompressHelper.read(d.data() + bytesRead, increments); + if (!decompressHelper.isValid()) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Decompression failed: %1") + .arg(decompressHelper.errorString())); + decompressHelper.clear(); + return; + } } d.resize(bytesRead); // we're synchronous so we're not calling this function again; reset the decompressHelper @@ -1142,13 +1149,15 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) // This can occur when downloading compressed data as some of the data may be the content // encoding's header. Don't emit anything for this. - if (previousBytesDownloaded == bytesDownloaded) { + if (lastReadyReadEmittedSize == bytesDownloaded) { if (readBufferMaxSize) emit q->readBufferFreed(dataSize); return; } + lastReadyReadEmittedSize = bytesDownloaded; - QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + const auto totalSizeOpt = QNetworkHeadersPrivate::toInt( + headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); emit q->readyRead(); // emit readyRead before downloadProgress in case this will cause events to be @@ -1156,8 +1165,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval && (!decompressHelper.isValid() || decompressHelper.isCountingBytes())) { downloadProgressSignalChoke.restart(); - emit q->downloadProgress(bytesDownloaded, - totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + emit q->downloadProgress(bytesDownloaded, totalSizeOpt.value_or(-1)); } } @@ -1229,13 +1237,12 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt // equal to "80", the port component value MUST be preserved; // otherwise, if the URI does not contain an explicit port // component, the UA MUST NOT add one. - url.setScheme(QLatin1String("https")); + url.setScheme("https"_L1); if (url.port() == 80) url.setPort(443); } - const bool isLessSafe = schemeBefore == QLatin1String("https") - && url.scheme() == QLatin1String("http"); + const bool isLessSafe = schemeBefore == "https"_L1 && url.scheme() == "http"_L1; if (httpRequest.redirectPolicy() == QNetworkRequest::NoLessSafeRedirectPolicy && isLessSafe) { error(QNetworkReply::InsecureRedirectError, @@ -1243,13 +1250,19 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt return; } + // If the original operation was a GET with a body and the status code is either + // 307 or 308 then keep the message body + const bool getOperationKeepsBody = (operation == QNetworkAccessManager::GetOperation) + && (httpStatus == 307 || httpStatus == 308); + redirectRequest = createRedirectRequest(originalRequest, url, maxRedirectsRemaining); operation = getRedirectOperation(operation, httpStatus); // Clear stale headers, the relevant ones get set again later httpRequest.clearHeaders(); - if (operation == QNetworkAccessManager::GetOperation - || operation == QNetworkAccessManager::HeadOperation) { + auto newHeaders = redirectRequest.headers(); + if ((operation == QNetworkAccessManager::GetOperation + || operation == QNetworkAccessManager::HeadOperation) && !getOperationKeepsBody) { // possibly changed from not-GET/HEAD to GET/HEAD, make sure to get rid of upload device uploadByteDevice.reset(); uploadByteDevicePosition = 0; @@ -1262,18 +1275,20 @@ void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int htt outgoingData = nullptr; outgoingDataBuffer.reset(); // We need to explicitly unset these headers so they're not reapplied to the httpRequest - redirectRequest.setHeader(QNetworkRequest::ContentLengthHeader, QVariant()); - redirectRequest.setHeader(QNetworkRequest::ContentTypeHeader, QVariant()); + newHeaders.removeAll(QHttpHeaders::WellKnownHeader::ContentLength); + newHeaders.removeAll(QHttpHeaders::WellKnownHeader::ContentType); } if (const QNetworkCookieJar *const cookieJar = manager->cookieJar()) { auto cookies = cookieJar->cookiesForUrl(url); if (!cookies.empty()) { - redirectRequest.setHeader(QNetworkRequest::KnownHeaders::CookieHeader, - QVariant::fromValue(cookies)); + auto cookieHeader = QNetworkHeadersPrivate::fromCookieList(cookies); + newHeaders.replaceOrAppend(QHttpHeaders::WellKnownHeader::Cookie, cookieHeader); } } + redirectRequest.setHeaders(std::move(newHeaders)); + if (httpRequest.redirectPolicy() != QNetworkRequest::UserVerifiedRedirectPolicy) followRedirect(); @@ -1286,8 +1301,7 @@ void QNetworkReplyHttpImplPrivate::followRedirect() Q_ASSERT(managerPrivate); decompressHelper.clear(); - rawHeaders.clear(); - cookedHeaders.clear(); + clearHeaders(); if (managerPrivate->thread) managerPrivate->thread->disconnect(); @@ -1296,6 +1310,8 @@ void QNetworkReplyHttpImplPrivate::followRedirect() q, [this]() { postRequest(redirectRequest); }, Qt::QueuedConnection); } +static constexpr QLatin1StringView locationHeader() noexcept { return "location"_L1; } + void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) { Q_Q(QNetworkReplyHttpImpl); @@ -1308,15 +1324,15 @@ void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) // What do we do about the caching of the HTML note? // The response to a 303 MUST NOT be cached, while the response to // all of the others is cacheable if the headers indicate it to be - QByteArray header = q->rawHeader("location"); + QByteArrayView header = q->headers().value(locationHeader()); QUrl url = QUrl(QString::fromUtf8(header)); if (!url.isValid()) - url = QUrl(QLatin1String(header)); + url = QUrl(QLatin1StringView(header)); q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url); } } -void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByteArray,QByteArray> > &hm, +void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QHttpHeaders &hm, int sc, const QString &rp, bool pu, QSharedPointer<char> db, qint64 contentLength, @@ -1335,7 +1351,7 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte // RFC6797, 8.1 // If an HTTP response is received over insecure transport, the UA MUST // ignore any present STS header field(s). - if (url.scheme() == QLatin1String("https") && managerPrivate->stsEnabled) + if (url.scheme() == "https"_L1 && managerPrivate->stsEnabled) managerPrivate->stsCache.updateFromHeaders(hm, url); #endif // Download buffer @@ -1349,45 +1365,40 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte q->setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); q->setAttribute(QNetworkRequest::Http2WasUsedAttribute, h2Used); + // A user having manually defined which encodings they accept is, for + // somwehat unknown (presumed legacy compatibility) reasons treated as + // disabling our decompression: + const bool autoDecompress = !request.headers().contains(QHttpHeaders::WellKnownHeader::AcceptEncoding); + const bool shouldDecompress = isCompressed && autoDecompress; // reconstruct the HTTP header - QList<QPair<QByteArray, QByteArray> > headerMap = hm; - QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(), - end = headerMap.constEnd(); - for (; it != end; ++it) { - QByteArray value = q->rawHeader(it->first); + auto h = q->headers(); + for (qsizetype i = 0; i < hm.size(); ++i) { + const auto key = hm.nameAt(i); + const auto originValue = hm.valueAt(i); // Reset any previous "location" header set in the reply. In case of // redirects, we don't want to 'append' multiple location header values, // rather we keep only the latest one - if (it->first.toLower() == "location") - value.clear(); - - if (isCompressed && !decompressHelper.isValid() - && it->first.compare("content-encoding", Qt::CaseInsensitive) == 0) { + if (key.compare(locationHeader(), Qt::CaseInsensitive) == 0) + h.removeAll(key); + if (shouldDecompress && !decompressHelper.isValid() && key == "content-encoding"_L1) { if (!synchronous) // with synchronous all the data is expected to be handled at once decompressHelper.setCountingBytesEnabled(true); - if (!decompressHelper.setEncoding(it->second)) { - // error occurred, error copied from QHttpNetworkConnectionPrivate::errorDetail - error(QNetworkReplyImpl::NetworkError::ProtocolFailure, - QCoreApplication::translate("QHttp", "Data corrupted")); + if (!decompressHelper.setEncoding(originValue)) { + error(QNetworkReplyImpl::NetworkError::UnknownContentError, + QCoreApplication::translate("QHttp", "Failed to initialize decompression: %1") + .arg(decompressHelper.errorString())); + return; } decompressHelper.setDecompressedSafetyCheckThreshold( request.decompressedSafetyCheckThreshold()); } - if (!value.isEmpty()) { - // Why are we appending values for headers which are already - // present? - if (it->first.compare("set-cookie", Qt::CaseInsensitive) == 0) - value += '\n'; - else - value += ", "; - } - value += it->second; - q->setRawHeader(it->first, value); + h.append(key, originValue); } + q->setHeaders(std::move(h)); q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); @@ -1402,14 +1413,11 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QList<QPair<QByte QAbstractNetworkCache *nc = managerPrivate->networkCache; if (nc) { QNetworkCacheMetaData metaData = nc->metaData(httpRequest.url()); - QNetworkHeadersPrivate cacheHeaders; - cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); - QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - it = cacheHeaders.findRawHeader("Cache-Control"); + auto value = metaData.headers().value(QHttpHeaders::WellKnownHeader::CacheControl); bool mustReValidate = false; - if (it != cacheHeaders.rawHeaders.constEnd()) { - QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) + if (!value.empty()) { + QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(value); + if (cacheControl.contains("must-revalidate"_ba)) mustReValidate = true; } if (!mustReValidate && sendCacheContents(metaData)) @@ -1647,16 +1655,21 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); q->setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); - QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders(); - QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), - end = rawHeaders.constEnd(); + QHttpHeaders cachedHeaders = metaData.headers(); + QHttpHeaders h = headers(); QUrl redirectUrl; - for ( ; it != end; ++it) { - if (httpRequest.isFollowRedirects() && - !it->first.compare("location", Qt::CaseInsensitive)) - redirectUrl = QUrl::fromEncoded(it->second); - setRawHeader(it->first, it->second); + for (qsizetype i = 0; i < cachedHeaders.size(); ++i) { + const auto name = cachedHeaders.nameAt(i); + const auto value = cachedHeaders.valueAt(i); + + if (httpRequest.isFollowRedirects() + && !name.compare(locationHeader(), Qt::CaseInsensitive)) { + redirectUrl = QUrl::fromEncoded(value); + } + + h.replaceOrAppend(name, value); } + setHeaders(std::move(h)); if (!isHttpRedirectResponse()) checkForRedirect(status); @@ -1690,33 +1703,43 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData return true; } +static auto caseInsensitiveCompare(QByteArrayView value) +{ + return [value](QByteArrayView element) + { + return value.compare(element, Qt::CaseInsensitive) == 0; + }; +} + +static bool isHopByHop(QByteArrayView header) +{ + constexpr QByteArrayView headers[] = { "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade"}; + return std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(header)); +} + QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const { Q_Q(const QNetworkReplyHttpImpl); QNetworkCacheMetaData metaData = oldMetaData; + QHttpHeaders cacheHeaders = metaData.headers(); + + const auto newHeaders = q->headers(); + for (qsizetype i = 0; i < newHeaders.size(); ++i) { + const auto name = newHeaders.nameAt(i); + const auto value = newHeaders.valueAt(i); - QNetworkHeadersPrivate cacheHeaders; - cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); - QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - - const QList<QByteArray> newHeaders = q->rawHeaderList(); - for (QByteArray header : newHeaders) { - QByteArray originalHeader = header; - header = header.toLower(); - bool hop_by_hop = - (header == "connection" - || header == "keep-alive" - || header == "proxy-authenticate" - || header == "proxy-authorization" - || header == "te" - || header == "trailers" - || header == "transfer-encoding" - || header == "upgrade"); - if (hop_by_hop) + if (isHopByHop(name)) continue; - if (header == "set-cookie") + if (name.compare("set-cookie", Qt::CaseInsensitive) == 0) continue; // for 4.6.0, we were planning to not store the date header in the @@ -1729,51 +1752,47 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe //continue; // Don't store Warning 1xx headers - if (header == "warning") { - QByteArray v = q->rawHeader(header); - if (v.length() == 3 - && v[0] == '1' - && v[1] >= '0' && v[1] <= '9' - && v[2] >= '0' && v[2] <= '9') + if (name.compare("warning", Qt::CaseInsensitive) == 0) { + if (value.size() == 3 + && value[0] == '1' + && isAsciiDigit(value[1]) + && isAsciiDigit(value[2])) continue; } - it = cacheHeaders.findRawHeader(header); - if (it != cacheHeaders.rawHeaders.constEnd()) { + if (cacheHeaders.contains(name)) { // Match the behavior of Firefox and assume Cache-Control: "no-transform" - if (header == "content-encoding" - || header == "content-range" - || header == "content-type") + constexpr QByteArrayView headers[]= + {"content-encoding", "content-range", "content-type"}; + if (std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(name))) continue; } // IIS has been known to send "Content-Length: 0" on 304 responses, so // ignore this too - if (header == "content-length" && statusCode == 304) + if (statusCode == 304 && name.compare("content-length", Qt::CaseInsensitive) == 0) continue; #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) - QByteArray n = q->rawHeader(header); - QByteArray o; - if (it != cacheHeaders.rawHeaders.constEnd()) - o = (*it).second; - if (n != o && header != "date") { - qDebug() << "replacing" << header; + QByteArrayView n = newHeaders.value(name); + QByteArrayView o = cacheHeaders.value(name); + if (n != o && name.compare("date", Qt::CaseInsensitive) != 0) { + qDebug() << "replacing" << name; qDebug() << "new" << n; qDebug() << "old" << o; } #endif - cacheHeaders.setRawHeader(originalHeader, q->rawHeader(header)); + cacheHeaders.replaceOrAppend(name, value); } - metaData.setRawHeaders(cacheHeaders.rawHeaders); + metaData.setHeaders(cacheHeaders); bool checkExpired = true; QHash<QByteArray, QByteArray> cacheControl; - it = cacheHeaders.findRawHeader("Cache-Control"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - cacheControl = parseHttpOptionHeader(it->second); - QByteArray maxAge = cacheControl.value("max-age"); + auto value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::CacheControl); + if (!value.empty()) { + cacheControl = parseHttpOptionHeader(value); + QByteArray maxAge = cacheControl.value("max-age"_ba); if (!maxAge.isEmpty()) { checkExpired = false; QDateTime dt = QDateTime::currentDateTimeUtc(); @@ -1782,16 +1801,18 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe } } if (checkExpired) { - it = cacheHeaders.findRawHeader("expires"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(it->second); + if (const auto value = cacheHeaders.value( + QHttpHeaders::WellKnownHeader::Expires); !value.isEmpty()) { + QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(value); metaData.setExpirationDate(expiredDateTime); } } - it = cacheHeaders.findRawHeader("last-modified"); - if (it != cacheHeaders.rawHeaders.constEnd()) - metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(it->second)); + if (const auto value = cacheHeaders.value( + QHttpHeaders::WellKnownHeader::LastModified); !value.isEmpty()) { + metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(value)); + } + bool canDiskCache; // only cache GET replies by default, all other replies (POST, PUT, DELETE) @@ -1800,7 +1821,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe canDiskCache = true; // HTTP/1.1. Check the Cache-Control header - if (cacheControl.contains("no-store")) + if (cacheControl.contains("no-store"_ba)) canDiskCache = false; // responses to POST might be cacheable @@ -1809,7 +1830,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe canDiskCache = false; // some pages contain "expires:" and "cache-control: no-cache" field, // so we only might cache POST requests if we get "cache-control: max-age ..." - if (cacheControl.contains("max-age")) + if (cacheControl.contains("max-age"_ba)) canDiskCache = true; // responses to PUT and DELETE are not cacheable @@ -1839,15 +1860,17 @@ bool QNetworkReplyHttpImplPrivate::canResume() const if (operation != QNetworkAccessManager::GetOperation) return false; + const auto h = q->headers(); + // Can only resume if server/resource supports Range header. - QByteArray acceptRangesheaderName("Accept-Ranges"); - if (!q->hasRawHeader(acceptRangesheaderName) || q->rawHeader(acceptRangesheaderName) == "none") + const auto acceptRanges = h.value(QHttpHeaders::WellKnownHeader::AcceptRanges); + if (acceptRanges.empty() || acceptRanges == "none") return false; // We only support resuming for byte ranges. - if (request.hasRawHeader("Range")) { - QByteArray range = request.rawHeader("Range"); - if (!range.startsWith("bytes=")) + const auto range = h.value(QHttpHeaders::WellKnownHeader::Range); + if (!range.empty()) { + if (!range.startsWith(bytesEqualPrefix())) return false; } @@ -1866,7 +1889,9 @@ void QNetworkReplyHttpImplPrivate::setResumeOffset(quint64 offset) void QNetworkReplyHttpImplPrivate::_q_startOperation() { - if (state == Working) // ensure this function is only being called once + // Ensure this function is only being called once, and not at all if we were + // cancelled + if (state >= Working) return; state = Working; @@ -1893,8 +1918,8 @@ void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead() // Needs to be done where sendCacheContents() (?) of HTTP is emitting // metaDataChanged ? - - QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + const auto totalSizeOpt = QNetworkHeadersPrivate::toInt( + headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); // emit readyRead before downloadProgress in case this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). @@ -1905,8 +1930,7 @@ void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead() if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) { downloadProgressSignalChoke.restart(); - emit q->downloadProgress(bytesDownloaded, - totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + emit q->downloadProgress(bytesDownloaded, totalSizeOpt.value_or(-1)); } } @@ -2026,9 +2050,9 @@ void QNetworkReplyHttpImplPrivate::setupTransferTimeout() Qt::QueuedConnection); } transferTimeout->stop(); - if (request.transferTimeout()) { + if (request.transferTimeoutAsDuration() > 0ms) { transferTimeout->setSingleShot(true); - transferTimeout->setInterval(request.transferTimeout()); + transferTimeout->setInterval(request.transferTimeoutAsDuration()); QMetaObject::invokeMethod(transferTimeout, "start", Qt::QueuedConnection); @@ -2093,11 +2117,13 @@ void QNetworkReplyHttpImplPrivate::finished() if (state == Finished || state == Aborted) return; - QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + const auto totalSizeOpt = QNetworkHeadersPrivate::toInt( + headers().value(QHttpHeaders::WellKnownHeader::ContentLength)); + const qint64 totalSize = totalSizeOpt.value_or(-1); // if we don't know the total size of or we received everything save the cache. // If the data is compressed then this is done in readData() - if ((totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize) + if ((totalSize == -1 || bytesDownloaded == totalSize) && !decompressHelper.isValid()) { completeCacheSave(); } @@ -2110,10 +2136,10 @@ void QNetworkReplyHttpImplPrivate::finished() state = Finished; q->setFinished(true); - if (totalSize.isNull() || totalSize == -1) { + if (totalSize == -1) { emit q->downloadProgress(bytesDownloaded, bytesDownloaded); } else { - emit q->downloadProgress(bytesDownloaded, totalSize.toLongLong()); + emit q->downloadProgress(bytesDownloaded, totalSize); } if (bytesUploaded == -1 && (outgoingData || outgoingDataBuffer)) @@ -2134,13 +2160,12 @@ void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, c Q_Q(QNetworkReplyHttpImpl); // Can't set and emit multiple errors. if (errorCode != QNetworkReply::NoError) { - qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."); + // But somewhat unavoidable if we have cancelled the request: + if (errorCode != QNetworkReply::OperationCanceledError) + qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."); return; } - if (decompressHelper.isValid()) - decompressHelper.clear(); // Just get rid of any data that might be stored - errorCode = code; q->setErrorString(errorMessage); @@ -2158,14 +2183,15 @@ void QNetworkReplyHttpImplPrivate::_q_metaDataChanged() // 1. do we have cookies? // 2. are we allowed to set them? Q_ASSERT(manager); - const auto it = cookedHeaders.constFind(QNetworkRequest::SetCookieHeader); - if (it != cookedHeaders.cend() + + const auto cookiesOpt = QNetworkHeadersPrivate::toSetCookieList( + headers().values(QHttpHeaders::WellKnownHeader::SetCookie)); + const auto cookies = cookiesOpt.value_or(QList<QNetworkCookie>()); + if (!cookies.empty() && request.attribute(QNetworkRequest::CookieSaveControlAttribute, QNetworkRequest::Automatic).toInt() == QNetworkRequest::Automatic) { QNetworkCookieJar *jar = manager->cookieJar(); if (jar) { - QList<QNetworkCookie> cookies = - qvariant_cast<QList<QNetworkCookie> >(it.value()); jar->setCookiesFromUrl(cookies, url); } } @@ -2229,3 +2255,5 @@ void QNetworkReplyHttpImplPrivate::completeCacheSave() } QT_END_NAMESPACE + +#include "moc_qnetworkreplyhttpimpl_p.cpp" |