QHttpHeaderParser: Allow larger fields but restrict total size

Our limit of 8k for a single header field was too small for certain
services which returned Location values or WWW-Authenticate challenges
longer than 8k.

Instead, taking inspiration from Chromium, we have a limit of 250k for
the full header itself. And increasing our field limit to 100k, which
would occupy almost half of what the total header allows.

Also took the opportunity to make it adjustable from the outside so we
can set more strict limits in other components.

Fixes: QTBUG-104132
Change-Id: Ibbe139445e79baaef30829cfbc9a59f884e96293
Reviewed-by: Ievgenii Meshcheriakov <ievgenii.meshcheriakov@qt.io>
(cherry picked from commit e3ea1d02e6a2cbc63e8ae6fff6ccae49258fe5a2)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

6 files changed, 183 insertions, 27 deletions

diff --git a/src/network/access/qhttpheaderparser.cpp b/src/network/access/qhttpheaderparser.cpp
index 345f548c51..8ed5592712 100644
--- a/src/network/access/qhttpheaderparser.cpp
+++ b/src/network/access/qhttpheaderparser.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2021 The Qt Company Ltd.
+// Copyright (C) 2022 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include "qhttpheaderparser_p.h"
@@ -7,12 +7,6 @@
 
 QT_BEGIN_NAMESPACE
 
-// both constants are taken from the default settings of Apache
-// see: http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfieldsize and
-// http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
-static const int MAX_HEADER_FIELD_SIZE = 8 * 1024;
-static const int MAX_HEADER_FIELDS = 100;
-
 QHttpHeaderParser::QHttpHeaderParser()
     : statusCode(100) // Required by tst_QHttpNetworkConnection::ignoresslerror(failure)
     , majorVersion(0)
@@ -55,19 +49,22 @@ bool QHttpHeaderParser::parseHeaders(QByteArrayView header)
     while (int tail = header.endsWith("\n\r\n") ? 2 : header.endsWith("\n\n") ? 1 : 0)
         header.chop(tail);
 
+    if (header.size() - (header.endsWith("\r\n") ? 2 : 1) > maxTotalSize)
+        return false;
+
     QList<QPair<QByteArray, QByteArray>> result;
     while (header.size()) {
         const int colon = header.indexOf(':');
         if (colon == -1) // if no colon check if empty headers
             return result.size() == 0 && (header == "\n" || header == "\r\n");
-        if (result.size() >= MAX_HEADER_FIELDS)
+        if (result.size() >= maxFieldCount)
             return false;
         QByteArrayView name = header.first(colon);
         if (!fieldNameCheck(name))
             return false;
         header = header.sliced(colon + 1);
         QByteArray value;
-        int valueSpace = MAX_HEADER_FIELD_SIZE - name.size() - 1;
+        qsizetype valueSpace = maxFieldSize - name.size() - 1;
         do {
             const int endLine = header.indexOf('\n');
             Q_ASSERT(endLine != -1);
@@ -84,7 +81,7 @@ bool QHttpHeaderParser::parseHeaders(QByteArrayView header)
             }
             header = header.sliced(endLine + 1);
         } while (hSpaceStart(header));
-        Q_ASSERT(name.size() + 1 + value.size() <= MAX_HEADER_FIELD_SIZE);
+        Q_ASSERT(name.size() + 1 + value.size() <= maxFieldSize);
         result.append(qMakePair(name.toByteArray(), value));
     }
 
diff --git a/src/network/access/qhttpheaderparser_p.h b/src/network/access/qhttpheaderparser_p.h
index 7b70b174bf..9b149570e0 100644
--- a/src/network/access/qhttpheaderparser_p.h
+++ b/src/network/access/qhttpheaderparser_p.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2021 The Qt Company Ltd.
+// Copyright (C) 2022 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #ifndef QHTTPHEADERPARSER_H
@@ -24,6 +24,25 @@
 
 QT_BEGIN_NAMESPACE
 
+namespace HeaderConstants {
+
+// We previously used 8K, which is common on server side, but it turned out to
+// not be enough for various uses. Historically Firefox used 10K as the limit of
+// a single field, but some Location headers and Authorization challenges can
+// get even longer. Other browsers, such as Chrome, instead have a limit on the
+// total size of all the headers (as well as extra limits on some of the
+// individual fields). We'll use 100K as our default limit, which would be a ridiculously large
+// header, with the possibility to override it where we need to.
+static constexpr int MAX_HEADER_FIELD_SIZE = 100 * 1024;
+// Taken from http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
+static constexpr int MAX_HEADER_FIELDS = 100;
+// Chromium has a limit on the total size of the header set to 256KB,
+// which is a reasonable default for QNetworkAccessManager.
+// https://stackoverflow.com/a/3436155
+static constexpr int MAX_TOTAL_HEADER_SIZE = 256 * 1024;
+
+}
+
 class Q_NETWORK_PRIVATE_EXPORT QHttpHeaderParser
 {
 public:
@@ -54,12 +73,25 @@ public:
     void removeHeaderField(const QByteArray &name);
     void clearHeaders();
 
+    void setMaxHeaderFieldSize(qsizetype size) { maxFieldSize = size; }
+    qsizetype maxHeaderFieldSize() const { return maxFieldSize; }
+
+    void setMaxTotalHeaderSize(qsizetype size) { maxTotalSize = size; }
+    qsizetype maxTotalHeaderSize() const { return maxTotalSize; }
+
+    void setMaxHeaderFields(qsizetype count) { maxFieldCount = count; }
+    qsizetype maxHeaderFields() const { return maxFieldCount; }
+
 private:
     QList<QPair<QByteArray, QByteArray> > fields;
     QString reasonPhrase;
     int statusCode;
     int majorVersion;
     int minorVersion;
+
+    qsizetype maxFieldSize = HeaderConstants::MAX_HEADER_FIELD_SIZE;
+    qsizetype maxTotalSize = HeaderConstants::MAX_TOTAL_HEADER_SIZE;
+    qsizetype maxFieldCount = HeaderConstants::MAX_HEADER_FIELDS;
 };
 
 
diff --git a/tests/auto/network/access/CMakeLists.txt b/tests/auto/network/access/CMakeLists.txt
index d225eb1299..3292bb1471 100644
--- a/tests/auto/network/access/CMakeLists.txt
+++ b/tests/auto/network/access/CMakeLists.txt
@@ -9,6 +9,7 @@ add_subdirectory(qnetworkreply)
 add_subdirectory(qnetworkcachemetadata)
 add_subdirectory(qabstractnetworkcache)
 if(QT_FEATURE_private_tests)
+    add_subdirectory(qhttpheaderparser)
     add_subdirectory(qhttpnetworkconnection)
     add_subdirectory(qhttpnetworkreply)
     add_subdirectory(hpack)
diff --git a/tests/auto/network/access/qhttpheaderparser/CMakeLists.txt b/tests/auto/network/access/qhttpheaderparser/CMakeLists.txt
new file mode 100644
index 0000000000..2d7d65d20c
--- /dev/null
+++ b/tests/auto/network/access/qhttpheaderparser/CMakeLists.txt
@@ -0,0 +1,11 @@
+
+if(NOT QT_FEATURE_private_tests)
+    return()
+endif()
+
+qt_internal_add_test(tst_qhttpheaderparser
+    SOURCES
+        tst_qhttpheaderparser.cpp
+    LIBRARIES
+        Qt::NetworkPrivate
+)
diff --git a/tests/auto/network/access/qhttpheaderparser/tst_qhttpheaderparser.cpp b/tests/auto/network/access/qhttpheaderparser/tst_qhttpheaderparser.cpp
new file mode 100644
index 0000000000..a1ea1c8ce7
--- /dev/null
+++ b/tests/auto/network/access/qhttpheaderparser/tst_qhttpheaderparser.cpp
@@ -0,0 +1,94 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QtTest/qtest.h>
+#include <QObject>
+#include <QtNetwork/private/qhttpheaderparser_p.h>
+
+class tst_QHttpHeaderParser : public QObject
+{
+    Q_OBJECT
+private Q_SLOTS:
+    void constructor();
+    void limitsSetters();
+
+    void adjustableLimits_data();
+    void adjustableLimits();
+
+    // general parsing tests can be found in tst_QHttpNetworkReply
+};
+
+void tst_QHttpHeaderParser::constructor()
+{
+    QHttpHeaderParser parser;
+    QCOMPARE(parser.getStatusCode(), 100);
+    QCOMPARE(parser.getMajorVersion(), 0);
+    QCOMPARE(parser.getMinorVersion(), 0);
+    QCOMPARE(parser.getReasonPhrase(), QByteArray());
+    QCOMPARE(parser.combinedHeaderValue("Location"), QByteArray());
+    QCOMPARE(parser.maxHeaderFields(), HeaderConstants::MAX_HEADER_FIELDS);
+    QCOMPARE(parser.maxHeaderFieldSize(), HeaderConstants::MAX_HEADER_FIELD_SIZE);
+    QCOMPARE(parser.maxTotalHeaderSize(), HeaderConstants::MAX_TOTAL_HEADER_SIZE);
+}
+
+void tst_QHttpHeaderParser::limitsSetters()
+{
+    QHttpHeaderParser parser;
+    parser.setMaxHeaderFields(10);
+    QCOMPARE(parser.maxHeaderFields(), 10);
+    parser.setMaxHeaderFieldSize(10);
+    QCOMPARE(parser.maxHeaderFieldSize(), 10);
+    parser.setMaxTotalHeaderSize(10);
+    QCOMPARE(parser.maxTotalHeaderSize(), 10);
+}
+
+void tst_QHttpHeaderParser::adjustableLimits_data()
+{
+    QTest::addColumn<qsizetype>("maxFieldCount");
+    QTest::addColumn<qsizetype>("maxFieldSize");
+    QTest::addColumn<qsizetype>("maxTotalSize");
+    QTest::addColumn<QByteArray>("headers");
+    QTest::addColumn<bool>("success");
+
+    // We pretend -1 means to not set a new limit.
+
+    QTest::newRow("maxFieldCount-pass") << qsizetype(10) << qsizetype(-1) << qsizetype(-1)
+                                        << QByteArray("Location: hi\r\n\r\n") << true;
+    QTest::newRow("maxFieldCount-fail") << qsizetype(1) << qsizetype(-1) << qsizetype(-1)
+                                        << QByteArray("Location: hi\r\nCookie: a\r\n\r\n") << false;
+
+    QTest::newRow("maxFieldSize-pass") << qsizetype(-1) << qsizetype(50) << qsizetype(-1)
+                                       << QByteArray("Location: hi\r\n\r\n") << true;
+    constexpr char cookieHeader[] = "Cookie: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+    static_assert(sizeof(cookieHeader) - 1 == 51);
+    QByteArray fullHeader = QByteArray("Location: hi\r\n") + cookieHeader;
+    QTest::newRow("maxFieldSize-fail") << qsizetype(-1) << qsizetype(50) << qsizetype(-1)
+                                       << (fullHeader + "\r\n\r\n") << false;
+
+    QTest::newRow("maxTotalSize-pass") << qsizetype(-1) << qsizetype(-1) << qsizetype(50)
+                                       << QByteArray("Location: hi\r\n\r\n") << true;
+    QTest::newRow("maxTotalSize-fail") << qsizetype(-1) << qsizetype(-1) << qsizetype(10)
+                                       << QByteArray("Location: hi\r\n\r\n") << false;
+}
+
+void tst_QHttpHeaderParser::adjustableLimits()
+{
+    QFETCH(qsizetype, maxFieldCount);
+    QFETCH(qsizetype, maxFieldSize);
+    QFETCH(qsizetype, maxTotalSize);
+    QFETCH(QByteArray, headers);
+    QFETCH(bool, success);
+
+    QHttpHeaderParser parser;
+    if (maxFieldCount != qsizetype(-1))
+        parser.setMaxHeaderFields(maxFieldCount);
+    if (maxFieldSize != qsizetype(-1))
+        parser.setMaxHeaderFieldSize(maxFieldSize);
+    if (maxTotalSize != qsizetype(-1))
+        parser.setMaxTotalHeaderSize(maxTotalSize);
+
+    QCOMPARE(parser.parseHeaders(headers), success);
+}
+
+QTEST_MAIN(tst_QHttpHeaderParser)
+#include "tst_qhttpheaderparser.moc"
diff --git a/tests/auto/network/access/qhttpnetworkreply/tst_qhttpnetworkreply.cpp b/tests/auto/network/access/qhttpnetworkreply/tst_qhttpnetworkreply.cpp
index bf19ff67c8..e36acc81da 100644
--- a/tests/auto/network/access/qhttpnetworkreply/tst_qhttpnetworkreply.cpp
+++ b/tests/auto/network/access/qhttpnetworkreply/tst_qhttpnetworkreply.cpp
@@ -1,10 +1,11 @@
-// Copyright (C) 2016 The Qt Company Ltd.
+// Copyright (C) 2022 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
 
 #include <QTest>
 #include <QtCore/QBuffer>
 #include <QtCore/QByteArray>
+#include <QtCore/QStringBuilder>
 
 #include "private/qhttpnetworkconnection_p.h"
 
@@ -83,12 +84,6 @@ void tst_QHttpNetworkReply::parseHeader()
     }
 }
 
-// both constants are taken from the default settings of Apache
-// see: http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfieldsize and
-// http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
-const int MAX_HEADER_FIELD_SIZE = 8 * 1024;
-const int MAX_HEADER_FIELDS = 100;
-
 void tst_QHttpNetworkReply::parseHeaderVerification_data()
 {
     QTest::addColumn<QByteArray>("headers");
@@ -106,36 +101,62 @@ void tst_QHttpNetworkReply::parseHeaderVerification_data()
     QTest::newRow("missing-colon-3")
             << QByteArray("Content-Encoding: gzip\r\nContent-Length\r\n") << false;
     QTest::newRow("header-field-too-long")
-            << (QByteArray("Content-Type: ") + QByteArray(MAX_HEADER_FIELD_SIZE, 'a')
-                + QByteArray("\r\n"))
+            << (QByteArray("Content-Type: ")
+                + QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE, 'a') + QByteArray("\r\n"))
             << false;
 
     QByteArray name = "Content-Type: ";
     QTest::newRow("max-header-field-size")
-            << (name + QByteArray(MAX_HEADER_FIELD_SIZE - name.size(), 'a') + QByteArray("\r\n"))
+            << (name + QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE - name.size(), 'a')
+                + QByteArray("\r\n"))
             << true;
 
     QByteArray tooManyHeaders = QByteArray("Content-Type: text/html; charset=utf-8\r\n")
-                                        .repeated(MAX_HEADER_FIELDS + 1);
+                                        .repeated(HeaderConstants::MAX_HEADER_FIELDS + 1);
     QTest::newRow("too-many-headers") << tooManyHeaders << false;
 
-    QByteArray maxHeaders =
-            QByteArray("Content-Type: text/html; charset=utf-8\r\n").repeated(MAX_HEADER_FIELDS);
+    QByteArray maxHeaders = QByteArray("Content-Type: text/html; charset=utf-8\r\n")
+                                    .repeated(HeaderConstants::MAX_HEADER_FIELDS);
     QTest::newRow("max-headers") << maxHeaders << true;
 
-    QByteArray firstValue(MAX_HEADER_FIELD_SIZE / 2, 'a');
+    QByteArray firstValue(HeaderConstants::MAX_HEADER_FIELD_SIZE / 2, 'a');
     constexpr int obsFold = 1;
     QTest::newRow("max-continuation-size")
             << (name + firstValue + QByteArray("\r\n ")
-                + QByteArray(MAX_HEADER_FIELD_SIZE - name.size() - firstValue.size() - obsFold, 'b')
+                + QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE - name.size()
+                                     - firstValue.size() - obsFold,
+                             'b')
                 + QByteArray("\r\n"))
             << true;
     QTest::newRow("too-long-continuation-size")
             << (name + firstValue + QByteArray("\r\n ")
-                + QByteArray(MAX_HEADER_FIELD_SIZE - name.size() - firstValue.size() - obsFold + 1,
+                + QByteArray(HeaderConstants::MAX_HEADER_FIELD_SIZE - name.size()
+                                     - firstValue.size() - obsFold + 1,
                              'b')
                 + QByteArray("\r\n"))
             << false;
+
+    auto appendLongHeaderElement = [](QByteArray &result, QByteArrayView name) {
+        const qsizetype size = result.size();
+        result += name;
+        result += ": ";
+        result.resize(size + HeaderConstants::MAX_HEADER_FIELD_SIZE, 'a');
+    };
+    QByteArray longHeader;
+    constexpr qsizetype TrailerLength = sizeof("\r\n\r\n") - 1; // we ignore the trailing newlines
+    longHeader.reserve(HeaderConstants::MAX_TOTAL_HEADER_SIZE + TrailerLength + 1);
+    appendLongHeaderElement(longHeader, "Location");
+    longHeader += "\r\n";
+    appendLongHeaderElement(longHeader, "WWW-Authenticate");
+    longHeader += "\r\nProxy-Authenticate: ";
+    longHeader.resize(HeaderConstants::MAX_TOTAL_HEADER_SIZE, 'a');
+    longHeader += "\r\n\r\n";
+
+    // Test with headers which are just large enough to fit our MAX_TOTAL_HEADER_SIZE limit:
+    QTest::newRow("total-header-close-to-max-size") << longHeader << true;
+    // Now add another character to make the total header size exceed the limit:
+    longHeader.insert(HeaderConstants::MAX_TOTAL_HEADER_SIZE - TrailerLength, 'a');
+    QTest::newRow("total-header-too-large") << longHeader << false;
 }
 
 void tst_QHttpNetworkReply::parseHeaderVerification()