QSslServer: Check that first byte is ClientHello

SecureTransport ignores any content that comes in until it is large
enough to be a handshake. So a plaintext client may be left hanging
while it is waiting for a response.

Pick-to: 6.4
Change-Id: I501ae61d89d516765c7ba5f0d916d9246fde5d4d
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

2 files changed, 82 insertions, 2 deletions

diff --git a/src/network/ssl/qsslserver.cpp b/src/network/ssl/qsslserver.cpp
index a26e79e315..1611046d0e 100644
--- a/src/network/ssl/qsslserver.cpp
+++ b/src/network/ssl/qsslserver.cpp
@@ -256,6 +256,8 @@ void QSslServer::incomingConnection(qintptr socket)
                         pSslSocket->deleteLater();
                 });
         connect(pSslSocket, &QSslSocket::encrypted, this, [this, pSslSocket]() {
+            Q_D(QSslServer);
+            d->removeSocketData(quintptr(pSslSocket));
             pSslSocket->disconnect(this);
             addPendingConnection(pSslSocket);
         });
@@ -278,10 +280,63 @@ void QSslServer::incomingConnection(qintptr socket)
                     Q_EMIT handshakeInterruptedOnError(pSslSocket, error);
                 });
 
-        Q_EMIT startedEncryptionHandshake(pSslSocket);
+        d_func()->initializeHandshakeProcess(pSslSocket);
+    }
+}
+
+void QSslServerPrivate::initializeHandshakeProcess(QSslSocket *socket)
+{
+    Q_Q(QSslServer);
+    QMetaObject::Connection readyRead = QObject::connect(
+            socket, &QSslSocket::readyRead, q, [this]() { checkClientHelloAndContinue(); });
+
+    QMetaObject::Connection destroyed =
+            QObject::connect(socket, &QSslSocket::destroyed, q, [this](QObject *obj) {
+                // This cast is not safe to use since the socket is inside the
+                // QObject dtor, but we only use the pointer value!
+                removeSocketData(quintptr(obj));
+            });
+    socketData.emplace(quintptr(socket), readyRead, destroyed);
+}
+
+// This function may be called while in the socket's QObject dtor, __never__ use
+// the socket for anything other than a lookup!
+void QSslServerPrivate::removeSocketData(quintptr socket)
+{
+    auto it = socketData.find(socket);
+    if (it != socketData.end()) {
+        it->disconnectSignals();
+        socketData.erase(it);
+    }
+}
 
-        pSslSocket->startServerEncryption();
+void QSslServerPrivate::checkClientHelloAndContinue()
+{
+    Q_Q(QSslServer);
+    QSslSocket *socket = qobject_cast<QSslSocket *>(q->sender());
+    if (Q_UNLIKELY(!socket) || socket->bytesAvailable() <= 0)
+        return;
+
+    char byte = '\0';
+    if (socket->peek(&byte, 1) != 1) {
+        socket->deleteLater();
+        return;
     }
+
+    auto it = socketData.find(quintptr(socket));
+    const bool foundData = it != socketData.end();
+    if (foundData && it->readyReadConnection)
+        QObject::disconnect(std::exchange(it->readyReadConnection, {}));
+
+    constexpr char CLIENT_HELLO = 0x16;
+    if (byte != CLIENT_HELLO) {
+        socket->disconnectFromHost();
+        socket->deleteLater();
+        return;
+    }
+
+    socket->startServerEncryption();
+    Q_EMIT q->startedEncryptionHandshake(socket);
 }
 
 QT_END_NAMESPACE
diff --git a/src/network/ssl/qsslserver_p.h b/src/network/ssl/qsslserver_p.h
index b4b6490ed4..3c7cce0355 100644
--- a/src/network/ssl/qsslserver_p.h
+++ b/src/network/ssl/qsslserver_p.h
@@ -16,8 +16,13 @@
 // We mean it.
 //
 
+#include <QtNetwork/private/qtnetworkglobal_p.h>
+
+#include <QtCore/qhash.h>
+
 #include <QtNetwork/QSslConfiguration>
 #include <QtNetwork/private/qtcpserver_p.h>
+#include <utility>
 
 QT_BEGIN_NAMESPACE
 
@@ -27,6 +32,26 @@ public:
     Q_DECLARE_PUBLIC(QSslServer)
 
     QSslServerPrivate();
+    void checkClientHelloAndContinue();
+    void initializeHandshakeProcess(QSslSocket *socket);
+    void removeSocketData(quintptr socket);
+
+    struct SocketData {
+        QMetaObject::Connection readyReadConnection;
+        QMetaObject::Connection destroyedConnection;
+
+        SocketData(QMetaObject::Connection readyRead, QMetaObject::Connection destroyed)
+            : readyReadConnection(readyRead), destroyedConnection(destroyed)
+        {
+        }
+
+        void disconnectSignals()
+        {
+            QObject::disconnect(std::exchange(readyReadConnection, {}));
+            QObject::disconnect(std::exchange(destroyedConnection, {}));
+        }
+    };
+    QHash<quintptr, SocketData> socketData;
 
     QSslConfiguration sslConfiguration;
 };