Implement protocol upgrade for HTTP/2 enabled requests

Without ALPN/NPN (== without OpenSSL) we can negotiate HTTP/2
using the protocol upgrade procedure as described by RFC7540.
Since there is no TLS handshake, after our http channel was connected
we first send an 'augmented' HTTP/1.1 request - its header
contains additional 'HTTP2-Settings' and 'Upgrade' (to 'h2c') fields.
If we receive reponse 101 (switch protocol) we re-create a protocol
handler and switch to HTTP/2.

Task-number: QTBUG-50955
Change-Id: I36e9985e06ba76edaf7fdb22bb43770f8d593c61
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

8 files changed, 149 insertions, 28 deletions

diff --git a/src/network/access/http2/http2protocol.cpp b/src/network/access/http2/http2protocol.cpp
index 7f788a6f42..9f05e926c9 100644
--- a/src/network/access/http2/http2protocol.cpp
+++ b/src/network/access/http2/http2protocol.cpp
@@ -37,9 +37,12 @@
 **
 ****************************************************************************/
 
+#include <QtCore/qbytearray.h>
 #include <QtCore/qstring.h>
 
+#include "private/qhttpnetworkrequest_p.h"
 #include "http2protocol_p.h"
+#include "http2frames_p.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -57,6 +60,37 @@ const char Http2clientPreface[clientPrefaceLength] =
      0x2e, 0x30, 0x0d, 0x0a, 0x0d, 0x0a,
      0x53, 0x4d, 0x0d, 0x0a, 0x0d, 0x0a};
 
+QByteArray qt_default_SETTINGS_to_Base64()
+{
+    FrameWriter frame(qt_default_SETTINGS_frame());
+    // SETTINGS frame's payload consists of pairs:
+    // 2-byte-identifier | 4-byte-value - multiple of 6.
+    // Also it's allowed to be empty.
+    Q_ASSERT(!(frame.payloadSize() % 6));
+    const char *src = reinterpret_cast<const char *>(&frame.rawFrameBuffer()[frameHeaderSize]);
+    const QByteArray wrapper(QByteArray::fromRawData(src, int(frame.payloadSize())));
+    // 3.2.1
+    // The content of the HTTP2-Settings header field is the payload
+    // of a SETTINGS frame (Section 6.5), encoded as a base64url string
+    // (that is, the URL- and filename-safe Base64 encoding described in
+    // Section 5 of [RFC4648], with any trailing '=' characters omitted).
+    return wrapper.toBase64(QByteArray::Base64UrlEncoding | QByteArray::OmitTrailingEquals);
+}
+
+void qt_add_ProtocolUpgradeRequest(QHttpNetworkRequest &request)
+{
+    // RFC 2616, 14.10
+    QByteArray value(request.headerField("Connection"));
+    if (value.size())
+        value += ", ";
+
+    value += "Upgrade, HTTP2-Settings";
+    request.setHeaderField("Connection", value);
+    // This we just (re)write.
+    request.setHeaderField("Upgrade", "h2c");
+    // This we just (re)write.
+    request.setHeaderField("HTTP2-Settings", qt_default_SETTINGS_to_Base64());
+}
 
 void qt_error(quint32 errorCode, QNetworkReply::NetworkError &error,
               QString &errorMessage)
@@ -151,6 +185,19 @@ QNetworkReply::NetworkError qt_error(quint32 errorCode)
     return error;
 }
 
+FrameWriter qt_default_SETTINGS_frame()
+{
+    // 6.5 SETTINGS
+    FrameWriter frame(FrameType::SETTINGS, FrameFlag::EMPTY, connectionStreamID);
+    // MAX frame size (16 kb), disable PUSH
+    frame.append(Settings::MAX_FRAME_SIZE_ID);
+    frame.append(quint32(maxFrameSize));
+    frame.append(Settings::ENABLE_PUSH_ID);
+    frame.append(quint32(0));
+
+    return frame;
+}
+
 }
 
 QT_END_NAMESPACE
diff --git a/src/network/access/http2/http2protocol_p.h b/src/network/access/http2/http2protocol_p.h
index 5c46949e23..e49e9f1218 100644
--- a/src/network/access/http2/http2protocol_p.h
+++ b/src/network/access/http2/http2protocol_p.h
@@ -59,6 +59,8 @@
 
 QT_BEGIN_NAMESPACE
 
+class QHttpNetworkRequest;
+class QByteArray;
 class QString;
 
 namespace Http2
@@ -128,6 +130,7 @@ enum Http2PredefinedParameters
 };
 
 extern const Q_AUTOTEST_EXPORT char Http2clientPreface[clientPrefaceLength];
+void qt_add_ProtocolUpgradeRequest(QHttpNetworkRequest &request);
 
 enum class FrameStatus
 {
@@ -166,6 +169,8 @@ void qt_error(quint32 errorCode, QNetworkReply::NetworkError &error, QString &er
 QString qt_error_string(quint32 errorCode);
 QNetworkReply::NetworkError qt_error(quint32 errorCode);
 
+class FrameWriter qt_default_SETTINGS_frame();
+
 }
 
 Q_DECLARE_LOGGING_CATEGORY(QT_HTTP2)
diff --git a/src/network/access/qhttp2protocolhandler.cpp b/src/network/access/qhttp2protocolhandler.cpp
index 937686920c..2cf44521eb 100644
--- a/src/network/access/qhttp2protocolhandler.cpp
+++ b/src/network/access/qhttp2protocolhandler.cpp
@@ -40,7 +40,7 @@
 #include "qhttpnetworkconnection_p.h"
 #include "qhttp2protocolhandler_p.h"
 
-#if !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#if !defined(QT_NO_HTTP)
 
 #include "http2/bitstreams_p.h"
 
@@ -54,6 +54,7 @@
 #include <QtCore/qurl.h>
 
 #include <algorithm>
+#include <utility>
 #include <vector>
 
 QT_BEGIN_NAMESPACE
@@ -133,6 +134,28 @@ QHttp2ProtocolHandler::QHttp2ProtocolHandler(QHttpNetworkConnectionChannel *chan
     continuedFrames.reserve(20);
 }
 
+QHttp2ProtocolHandler::QHttp2ProtocolHandler(QHttpNetworkConnectionChannel *channel,
+                                             const HttpMessagePair &message)
+    : QAbstractProtocolHandler(channel),
+      prefaceSent(false),
+      waitingForSettingsACK(false),
+      decoder(HPack::FieldLookupTable::DefaultSize),
+      encoder(HPack::FieldLookupTable::DefaultSize, true)
+{
+    // That's a protocol upgrade scenario - 3.2.
+    //
+    // We still have to send settings and the preface
+    // (though SETTINGS was a part of the first HTTP/1.1
+    // request "HTTP2-Settings" field).
+    //
+    // We pass 'false' for upload data, this was done by HTTP/1.1 protocol
+    // handler for us while sending the first request.
+    const quint32 initialStreamID = createNewStream(message, false);
+    Q_ASSERT(initialStreamID == 1);
+    Stream &stream = activeStreams[initialStreamID];
+    stream.state = Stream::halfClosedLocal;
+}
+
 void QHttp2ProtocolHandler::_q_uploadDataReadyRead()
 {
     auto data = qobject_cast<QNonContiguousByteDevice *>(sender());
@@ -247,7 +270,7 @@ bool QHttp2ProtocolHandler::sendRequest()
     auto it = requests.begin();
     m_channel->state = QHttpNetworkConnectionChannel::WritingState;
     for (quint32 i = 0; i < streamsToUse; ++i) {
-        const qint32 newStreamID = createNewStream(*it);
+        const qint32 newStreamID = createNewStream(*it, true /* upload data */);
         if (!newStreamID) {
             // TODO: actually we have to open a new connection.
             qCCritical(QT_HTTP2, "sendRequest: out of stream IDs");
@@ -278,7 +301,6 @@ bool QHttp2ProtocolHandler::sendRequest()
     return true;
 }
 
-
 bool QHttp2ProtocolHandler::sendClientPreface()
 {
      // 3.5 HTTP/2 Connection Preface
@@ -293,12 +315,8 @@ bool QHttp2ProtocolHandler::sendClientPreface()
         return false;
 
     // 6.5 SETTINGS
-    outboundFrame.start(FrameType::SETTINGS, FrameFlag::EMPTY, Http2::connectionStreamID);
-    // MAX frame size (16 kb), disable PUSH
-    outboundFrame.append(Settings::MAX_FRAME_SIZE_ID);
-    outboundFrame.append(quint32(Http2::maxFrameSize));
-    outboundFrame.append(Settings::ENABLE_PUSH_ID);
-    outboundFrame.append(quint32(0));
+    outboundFrame = Http2::qt_default_SETTINGS_frame();
+    Q_ASSERT(outboundFrame.payloadSize());
 
     if (!outboundFrame.write(*m_socket))
         return false;
@@ -1022,7 +1040,8 @@ void QHttp2ProtocolHandler::finishStreamWithError(Stream &stream, QNetworkReply:
     emit httpReply->finishedWithError(error, message);
 }
 
-quint32 QHttp2ProtocolHandler::createNewStream(const HttpMessagePair &message)
+quint32 QHttp2ProtocolHandler::createNewStream(const HttpMessagePair &message,
+                                               bool uploadData)
 {
     const qint32 newStreamID = allocateStreamID();
     if (!newStreamID)
@@ -1043,10 +1062,12 @@ quint32 QHttp2ProtocolHandler::createNewStream(const HttpMessagePair &message)
                            streamInitialSendWindowSize,
                            streamInitialRecvWindowSize);
 
-    if (auto src = newStream.data()) {
-        connect(src, SIGNAL(readyRead()), this,
-                SLOT(_q_uploadDataReadyRead()), Qt::QueuedConnection);
-        src->setProperty("HTTP2StreamID", newStreamID);
+    if (uploadData) {
+        if (auto src = newStream.data()) {
+            connect(src, SIGNAL(readyRead()), this,
+                    SLOT(_q_uploadDataReadyRead()), Qt::QueuedConnection);
+            src->setProperty("HTTP2StreamID", newStreamID);
+        }
     }
 
     activeStreams.insert(newStreamID, newStream);
@@ -1214,4 +1235,4 @@ void QHttp2ProtocolHandler::closeSession()
 
 QT_END_NAMESPACE
 
-#endif // !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#endif // !defined(QT_NO_HTTP)
diff --git a/src/network/access/qhttp2protocolhandler_p.h b/src/network/access/qhttp2protocolhandler_p.h
index b146e37dd3..e41b1360bf 100644
--- a/src/network/access/qhttp2protocolhandler_p.h
+++ b/src/network/access/qhttp2protocolhandler_p.h
@@ -55,7 +55,7 @@
 #include <private/qabstractprotocolhandler_p.h>
 #include <private/qhttpnetworkrequest_p.h>
 
-#if !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#if !defined(QT_NO_HTTP)
 
 #include "http2/http2protocol_p.h"
 #include "http2/http2streams_p.h"
@@ -81,7 +81,15 @@ class QHttp2ProtocolHandler : public QObject, public QAbstractProtocolHandler
     Q_OBJECT
 
 public:
+    // "TLS + ALPN/NPN" case:
     QHttp2ProtocolHandler(QHttpNetworkConnectionChannel *channel);
+    // HTTP2 without TLS - the first request was sent as an HTTP/1.1 request
+    // with Upgrade:h2c header. That serves as an implicit HTTP/2 request
+    // on a stream with ID 1 (it will be created in this ctor in a
+    // 'half-closed-local' state); reply, if server supports HTTP/2,
+    // will be HTTP/2 frame(s):
+    QHttp2ProtocolHandler(QHttpNetworkConnectionChannel *channel,
+                          const HttpMessagePair &message);
 
     QHttp2ProtocolHandler(const QHttp2ProtocolHandler &rhs) = delete;
     QHttp2ProtocolHandler(QHttp2ProtocolHandler &&rhs) = delete;
@@ -133,7 +141,7 @@ private:
                                const QString &message);
 
     // Stream's lifecycle management:
-    quint32 createNewStream(const HttpMessagePair &message);
+    quint32 createNewStream(const HttpMessagePair &message, bool uploadData);
     void addToSuspended(Stream &stream);
     void markAsReset(quint32 streamID);
     quint32 popStreamToResume();
@@ -202,6 +210,6 @@ private:
 
 QT_END_NAMESPACE
 
-#endif // !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#endif // !defined(QT_NO_HTTP)
 
 #endif
diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 3a780f636b..3d35fe5f04 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -50,6 +50,7 @@
 #include <private/qhttp2protocolhandler_p.h>
 #include <private/qhttpprotocolhandler_p.h>
 #include <private/qspdyprotocolhandler_p.h>
+#include <private/http2protocol_p.h>
 
 #ifndef QT_NO_SSL
 #    include <private/qsslsocket_p.h>
@@ -180,6 +181,9 @@ void QHttpNetworkConnectionChannel::init()
            sslSocket->setSslConfiguration(sslConfiguration);
     } else {
 #endif // QT_NO_SSL
+        // Even if connection->connectionType is ConnectionTypeHTTP2,
+        // we first start as HTTP/1.1, asking for a protocol upgrade
+        // in the first response.
         protocolHandler.reset(new QHttpProtocolHandler(this));
 #ifndef QT_NO_SSL
     }
@@ -835,6 +839,16 @@ void QHttpNetworkConnectionChannel::_q_connected()
 #endif
     } else {
         state = QHttpNetworkConnectionChannel::IdleState;
+        if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
+            Q_ASSERT(spdyRequestsToSend.size());
+            auto it = spdyRequestsToSend.begin();
+            // Let's inject some magic fields, requesting a protocol upgrade:
+            Http2::qt_add_ProtocolUpgradeRequest(it->first);
+            connection->d_func()->requeueRequest(*it);
+            // Remove it, we never send it again as HTTP/2.
+            spdyRequestsToSend.erase(it);
+        }
+
         if (!reply)
             connection->d_func()->dequeueRequest(socket);
         if (reply)
@@ -972,9 +986,12 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
         }
     } while (!connection->d_func()->highPriorityQueue.isEmpty()
              || !connection->d_func()->lowPriorityQueue.isEmpty());
+
+    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2
 #ifndef QT_NO_SSL
-    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY ||
-        connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
+        || connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY
+#endif
+        ) {
         QList<HttpMessagePair> spdyPairs = spdyRequestsToSend.values();
         for (int a = 0; a < spdyPairs.count(); ++a) {
             // emit error for all replies
@@ -983,7 +1000,6 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
             emit currentReply->finishedWithError(errorCode, errorString);
         }
     }
-#endif // QT_NO_SSL
 
     // send the next request
     QMetaObject::invokeMethod(that, "_q_startNextRequest", Qt::QueuedConnection);
@@ -1002,23 +1018,31 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
     }
 }
 
+void QHttpNetworkConnectionChannel::_q_protocolSwitch()
+{
+    Q_ASSERT(connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2);
+    Q_ASSERT(reply);
+    Q_ASSERT(reply->statusCode() == 101);
+    protocolHandler.reset(new QHttp2ProtocolHandler(this, HttpMessagePair(request, reply)));
+    protocolHandler->_q_receiveReply();
+}
+
 #ifndef QT_NO_NETWORKPROXY
 void QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator* auth)
 {
+    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2
 #ifndef QT_NO_SSL
-    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY ||
-        connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
+        || connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY
+#endif
+    ) {
         connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
     } else { // HTTP
-#endif // QT_NO_SSL
         // Need to dequeue the request before we can emit the error.
         if (!reply)
             connection->d_func()->dequeueRequest(socket);
         if (reply)
             connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
-#ifndef QT_NO_SSL
     }
-#endif // QT_NO_SSL
 }
 #endif
 
diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h
index d7d5d86a7a..a20cc1beb8 100644
--- a/src/network/access/qhttpnetworkconnectionchannel_p.h
+++ b/src/network/access/qhttpnetworkconnectionchannel_p.h
@@ -121,11 +121,14 @@ public:
     bool authenticationCredentialsSent;
     bool proxyCredentialsSent;
     QScopedPointer<QAbstractProtocolHandler> protocolHandler;
+    // SPDY or HTTP/2 requests; SPDY is TLS-only, but
+    // HTTP/2 can be cleartext also, that's why it's
+    // outside of QT_NO_SSL section. Sorted by priority:
+    QMultiMap<int, HttpMessagePair> spdyRequestsToSend;
 #ifndef QT_NO_SSL
     bool ignoreAllSslErrors;
     QList<QSslError> ignoreSslErrorsList;
     QSslConfiguration sslConfiguration;
-    QMultiMap<int, HttpMessagePair> spdyRequestsToSend; // sorted by priority
     void ignoreSslErrors();
     void ignoreSslErrors(const QList<QSslError> &errors);
     void setSslConfiguration(const QSslConfiguration &config);
@@ -192,6 +195,7 @@ public:
     void _q_disconnected(); // disconnected from host
     void _q_connected(); // start sending request
     void _q_error(QAbstractSocket::SocketError); // error from socket
+    void _q_protocolSwitch(); // HTTP/2 was negotiated to replace HTTP/1.1
 #ifndef QT_NO_NETWORKPROXY
     void _q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth); // from transparent proxy
 #endif
diff --git a/src/network/access/qhttpprotocolhandler.cpp b/src/network/access/qhttpprotocolhandler.cpp
index b486b75449..ab136af083 100644
--- a/src/network/access/qhttpprotocolhandler.cpp
+++ b/src/network/access/qhttpprotocolhandler.cpp
@@ -129,6 +129,15 @@ void QHttpProtocolHandler::_q_receiveReply()
                 } else {
                     replyPrivate->autoDecompress = false;
                 }
+                if (m_connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
+                    if (replyPrivate->statusCode == 101) {
+                        QMetaObject::invokeMethod(m_channel, "_q_protocolSwitch", Qt::QueuedConnection);
+                        return;
+                    }
+
+                    // HTTP/2 is not supported? TODO - but can it be something else?
+                    m_channel->requeueSpdyRequests();
+                }
                 if (replyPrivate->statusCode == 100) {
                     replyPrivate->clearHttpLayerInformation();
                     replyPrivate->state = QHttpNetworkReplyPrivate::ReadingStatusState;
diff --git a/src/network/access/qhttpthreaddelegate.cpp b/src/network/access/qhttpthreaddelegate.cpp
index e16519c2f2..8b200ebc04 100644
--- a/src/network/access/qhttpthreaddelegate.cpp
+++ b/src/network/access/qhttpthreaddelegate.cpp
@@ -286,9 +286,12 @@ void QHttpThreadDelegate::startRequest()
 
     QHttpNetworkConnection::ConnectionType connectionType
             = QHttpNetworkConnection::ConnectionTypeHTTP;
+
+    if (httpRequest.isHTTP2Allowed())
+        connectionType = QHttpNetworkConnection::ConnectionTypeHTTP2;
+
 #ifndef QT_NO_SSL
     if (httpRequest.isHTTP2Allowed() && ssl) {
-        connectionType = QHttpNetworkConnection::ConnectionTypeHTTP2;
         QList<QByteArray> protocols;
         protocols << QSslConfiguration::ALPNProtocolHTTP2
                   << QSslConfiguration::NextProtocolHttp1_1;