Merge remote-tracking branch 'origin/5.4' into merge5.5

Conflicts:
	src/corelib/global/qglobal.h
	src/corelib/io/qnoncontiguousbytedevice_p.h
	src/gui/image/qjpeghandler.cpp
	src/network/access/qhttpthreaddelegate_p.h
	tests/auto/corelib/io/qlockfile/tst_qlockfile.cpp
	tests/auto/widgets/widgets/qmenubar/BLACKLIST

Change-Id: I01de8c1c28efcedfd7953d05025f54802dc08ab3

6 files changed, 88 insertions, 24 deletions

diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 75888c0062..ef40d8cb2a 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -106,15 +106,19 @@ void QHttpNetworkConnectionChannel::init()
     socket->setProxy(QNetworkProxy::NoProxy);
 #endif
 
+    // We want all signals (except the interactive ones) be connected as QueuedConnection
+    // because else we're falling into cases where we recurse back into the socket code
+    // and mess up the state. Always going to the event loop (and expecting that when reading/writing)
+    // is safer.
     QObject::connect(socket, SIGNAL(bytesWritten(qint64)),
                      this, SLOT(_q_bytesWritten(qint64)),
-                     Qt::DirectConnection);
+                     Qt::QueuedConnection);
     QObject::connect(socket, SIGNAL(connected()),
                      this, SLOT(_q_connected()),
-                     Qt::DirectConnection);
+                     Qt::QueuedConnection);
     QObject::connect(socket, SIGNAL(readyRead()),
                      this, SLOT(_q_readyRead()),
-                     Qt::DirectConnection);
+                     Qt::QueuedConnection);
 
     // The disconnected() and error() signals may already come
     // while calling connectToHost().
@@ -143,7 +147,7 @@ void QHttpNetworkConnectionChannel::init()
         // won't be a sslSocket if encrypt is false
         QObject::connect(sslSocket, SIGNAL(encrypted()),
                          this, SLOT(_q_encrypted()),
-                         Qt::DirectConnection);
+                         Qt::QueuedConnection);
         QObject::connect(sslSocket, SIGNAL(sslErrors(QList<QSslError>)),
                          this, SLOT(_q_sslErrors(QList<QSslError>)),
                          Qt::DirectConnection);
@@ -152,7 +156,7 @@ void QHttpNetworkConnectionChannel::init()
                          Qt::DirectConnection);
         QObject::connect(sslSocket, SIGNAL(encryptedBytesWritten(qint64)),
                          this, SLOT(_q_encryptedBytesWritten(qint64)),
-                         Qt::DirectConnection);
+                         Qt::QueuedConnection);
 
         if (ignoreAllSslErrors)
             sslSocket->ignoreSslErrors();
@@ -189,8 +193,11 @@ void QHttpNetworkConnectionChannel::close()
     // pendingEncrypt must only be true in between connected and encrypted states
     pendingEncrypt = false;
 
-    if (socket)
+    if (socket) {
+        // socket can be 0 since the host lookup is done from qhttpnetworkconnection.cpp while
+        // there is no socket yet.
         socket->close();
+    }
 }
 
 
@@ -356,6 +363,14 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
         }
         return false;
     }
+
+    // This code path for ConnectedState
+    if (pendingEncrypt) {
+        // Let's only be really connected when we have received the encrypted() signal. Else the state machine seems to mess up
+        // and corrupt the things sent to the server.
+        return false;
+    }
+
     return true;
 }
 
@@ -662,6 +677,12 @@ bool QHttpNetworkConnectionChannel::isSocketReading() const
 void QHttpNetworkConnectionChannel::_q_bytesWritten(qint64 bytes)
 {
     Q_UNUSED(bytes);
+    if (ssl) {
+        // In the SSL case we want to send data from encryptedBytesWritten signal since that one
+        // is the one going down to the actual network, not only into some SSL buffer.
+        return;
+    }
+
     // bytes have been written to the socket. write even more of them :)
     if (isSocketWriting())
         sendRequest();
@@ -737,7 +758,7 @@ void QHttpNetworkConnectionChannel::_q_connected()
 
     // ### FIXME: if the server closes the connection unexpectedly, we shouldn't send the same broken request again!
     //channels[i].reconnectAttempts = 2;
-    if (pendingEncrypt) {
+    if (ssl || pendingEncrypt) { // FIXME: Didn't work properly with pendingEncrypt only, we should refactor this into an EncrypingState
 #ifndef QT_NO_SSL
         if (connection->sslContext().isNull()) {
             // this socket is making the 1st handshake for this connection,
diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h
index 6872fcb691..16d6c3b40f 100644
--- a/src/network/access/qhttpnetworkconnectionchannel_p.h
+++ b/src/network/access/qhttpnetworkconnectionchannel_p.h
@@ -83,6 +83,8 @@ typedef QPair<QHttpNetworkRequest, QHttpNetworkReply*> HttpMessagePair;
 class QHttpNetworkConnectionChannel : public QObject {
     Q_OBJECT
 public:
+    // TODO: Refactor this to add an EncryptingState (and remove pendingEncrypt).
+    // Also add an Unconnected state so IdleState does not have double meaning.
     enum ChannelState {
         IdleState = 0,          // ready to send request
         ConnectingState = 1,    // connecting to host
diff --git a/src/network/access/qhttpprotocolhandler.cpp b/src/network/access/qhttpprotocolhandler.cpp
index eac261bc80..55187755bf 100644
--- a/src/network/access/qhttpprotocolhandler.cpp
+++ b/src/network/access/qhttpprotocolhandler.cpp
@@ -368,6 +368,13 @@ bool QHttpProtocolHandler::sendRequest()
                 // nothing to read currently, break the loop
                 break;
             } else {
+                if (m_channel->written != uploadByteDevice->pos()) {
+                    // Sanity check. This was useful in tracking down an upload corruption.
+                    qWarning() << "QHttpProtocolHandler: Internal error in sendRequest. Expected to write at position" << m_channel->written << "but read device is at" << uploadByteDevice->pos();
+                    Q_ASSERT(m_channel->written == uploadByteDevice->pos());
+                    m_connection->d_func()->emitReplyError(m_socket, m_reply, QNetworkReply::ProtocolFailure);
+                    return false;
+                }
                 qint64 currentWriteSize = m_socket->write(readPointer, currentReadSize);
                 if (currentWriteSize == -1 || currentWriteSize != currentReadSize) {
                     // socket broke down
diff --git a/src/network/access/qhttpthreaddelegate_p.h b/src/network/access/qhttpthreaddelegate_p.h
index ff919b11c5..9573d500ea 100644
--- a/src/network/access/qhttpthreaddelegate_p.h
+++ b/src/network/access/qhttpthreaddelegate_p.h
@@ -189,6 +189,7 @@ protected:
     QByteArray m_dataArray;
     bool m_atEnd;
     qint64 m_size;
+    qint64 m_pos; // to match calls of haveDataSlot with the expected position
 public:
     QNonContiguousByteDeviceThreadForwardImpl(bool aE, qint64 s)
         : QNonContiguousByteDevice(),
@@ -196,7 +197,8 @@ public:
           m_amount(0),
           m_data(0),
           m_atEnd(aE),
-          m_size(s)
+          m_size(s),
+          m_pos(0)
     {
     }
 
@@ -204,6 +206,11 @@ public:
     {
     }
 
+    qint64 pos() Q_DECL_OVERRIDE
+    {
+        return m_pos;
+    }
+
     const char* readPointer(qint64 maximumLength, qint64 &len) Q_DECL_OVERRIDE
     {
         if (m_amount > 0) {
@@ -231,11 +238,10 @@ public:
 
         m_amount -= a;
         m_data += a;
+        m_pos += a;
 
-        // To main thread to inform about our state
-        emit processedData(a);
-
-        // FIXME possible optimization, already ask user thread for some data
+        // To main thread to inform about our state. The m_pos will be sent as a sanity check.
+        emit processedData(m_pos, a);
 
         return true;
     }
@@ -252,10 +258,21 @@ public:
     {
         m_amount = 0;
         m_data = 0;
+        m_dataArray.clear();
+
+        if (wantDataPending) {
+            // had requested the user thread to send some data (only 1 in-flight at any moment)
+            wantDataPending = false;
+        }
 
         // Communicate as BlockingQueuedConnection
         bool b = false;
         emit resetData(&b);
+        if (b) {
+            // the reset succeeded, we're at pos 0 again
+            m_pos = 0;
+            // the HTTP code will anyway abort the request if !b.
+        }
         return b;
     }
 
@@ -266,8 +283,13 @@ public:
 
 public slots:
     // From user thread:
-    void haveDataSlot(QByteArray dataArray, bool dataAtEnd, qint64 dataSize)
+    void haveDataSlot(qint64 pos, QByteArray dataArray, bool dataAtEnd, qint64 dataSize)
     {
+        if (pos != m_pos) {
+            // Sometimes when re-sending a request in the qhttpnetwork* layer there is a pending haveData from the
+            // user thread on the way to us. We need to ignore it since it is the data for the wrong(later) chunk.
+            return;
+        }
         wantDataPending = false;
 
         m_dataArray = dataArray;
@@ -287,7 +309,7 @@ signals:
 
     // to main thread:
     void wantData(qint64);
-    void processedData(qint64);
+    void processedData(qint64 pos, qint64 amount);
     void resetData(bool *b);
 };
 
diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp
index c92c360420..c445677792 100644
--- a/src/network/access/qnetworkreplyhttpimpl.cpp
+++ b/src/network/access/qnetworkreplyhttpimpl.cpp
@@ -424,6 +424,7 @@ QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate()
     , synchronous(false)
     , state(Idle)
     , statusCode(0)
+    , uploadByteDevicePosition(false)
     , uploadDeviceChoking(false)
     , outgoingData(0)
     , bytesUploaded(-1)
@@ -867,9 +868,9 @@ void QNetworkReplyHttpImplPrivate::postRequest()
                              q, SLOT(uploadByteDeviceReadyReadSlot()),
                              Qt::QueuedConnection);
 
-            // From main thread to user thread:
-            QObject::connect(q, SIGNAL(haveUploadData(QByteArray,bool,qint64)),
-                             forwardUploadDevice, SLOT(haveDataSlot(QByteArray,bool,qint64)), Qt::QueuedConnection);
+            // From user thread to http thread:
+            QObject::connect(q, SIGNAL(haveUploadData(qint64,QByteArray,bool,qint64)),
+                             forwardUploadDevice, SLOT(haveDataSlot(qint64,QByteArray,bool,qint64)), Qt::QueuedConnection);
             QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()),
                              forwardUploadDevice, SIGNAL(readyRead()),
                              Qt::QueuedConnection);
@@ -877,8 +878,8 @@ void QNetworkReplyHttpImplPrivate::postRequest()
             // From http thread to user thread:
             QObject::connect(forwardUploadDevice, SIGNAL(wantData(qint64)),
                              q, SLOT(wantUploadDataSlot(qint64)));
-            QObject::connect(forwardUploadDevice, SIGNAL(processedData(qint64)),
-                             q, SLOT(sentUploadDataSlot(qint64)));
+            QObject::connect(forwardUploadDevice,SIGNAL(processedData(qint64, qint64)),
+                             q, SLOT(sentUploadDataSlot(qint64,qint64)));
             QObject::connect(forwardUploadDevice, SIGNAL(resetData(bool*)),
                     q, SLOT(resetUploadDataSlot(bool*)),
                     Qt::BlockingQueuedConnection); // this is the only one with BlockingQueued!
@@ -1278,12 +1279,22 @@ void QNetworkReplyHttpImplPrivate::replyPreSharedKeyAuthenticationRequiredSlot(Q
 void QNetworkReplyHttpImplPrivate::resetUploadDataSlot(bool *r)
 {
     *r = uploadByteDevice->reset();
+    if (*r) {
+        // reset our own position which is used for the inter-thread communication
+        uploadByteDevicePosition = 0;
+    }
 }
 
 // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
-void QNetworkReplyHttpImplPrivate::sentUploadDataSlot(qint64 amount)
+void QNetworkReplyHttpImplPrivate::sentUploadDataSlot(qint64 pos, qint64 amount)
 {
+    if (uploadByteDevicePosition + amount != pos) {
+        // Sanity check, should not happen.
+        error(QNetworkReply::UnknownNetworkError, QString());
+        return;
+    }
     uploadByteDevice->advanceReadPointer(amount);
+    uploadByteDevicePosition += amount;
 }
 
 // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
@@ -1308,7 +1319,7 @@ void QNetworkReplyHttpImplPrivate::wantUploadDataSlot(qint64 maxSize)
     QByteArray dataArray(data, currentUploadDataLength);
 
     // Communicate back to HTTP thread
-    emit q->haveUploadData(dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size());
+    emit q->haveUploadData(uploadByteDevicePosition, dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size());
 }
 
 void QNetworkReplyHttpImplPrivate::uploadByteDeviceReadyReadSlot()
diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h
index afe807ea59..b009092196 100644
--- a/src/network/access/qnetworkreplyhttpimpl_p.h
+++ b/src/network/access/qnetworkreplyhttpimpl_p.h
@@ -121,7 +121,7 @@ public:
 
     Q_PRIVATE_SLOT(d_func(), void resetUploadDataSlot(bool *r))
     Q_PRIVATE_SLOT(d_func(), void wantUploadDataSlot(qint64))
-    Q_PRIVATE_SLOT(d_func(), void sentUploadDataSlot(qint64))
+    Q_PRIVATE_SLOT(d_func(), void sentUploadDataSlot(qint64,qint64))
     Q_PRIVATE_SLOT(d_func(), void uploadByteDeviceReadyReadSlot())
     Q_PRIVATE_SLOT(d_func(), void emitReplyUploadProgress(qint64, qint64))
     Q_PRIVATE_SLOT(d_func(), void _q_cacheSaveDeviceAboutToClose())
@@ -145,7 +145,7 @@ signals:
 
     void startHttpRequestSynchronously();
 
-    void haveUploadData(QByteArray dataArray, bool dataAtEnd, qint64 dataSize);
+    void haveUploadData(const qint64 pos, QByteArray dataArray, bool dataAtEnd, qint64 dataSize);
 };
 
 class QNetworkReplyHttpImplPrivate: public QNetworkReplyPrivate
@@ -196,6 +196,7 @@ public:
     // upload
     QNonContiguousByteDevice* createUploadByteDevice();
     QSharedPointer<QNonContiguousByteDevice> uploadByteDevice;
+    qint64 uploadByteDevicePosition;
     bool uploadDeviceChoking; // if we couldn't readPointer() any data at the moment
     QIODevice *outgoingData;
     QSharedPointer<QRingBuffer> outgoingDataBuffer;
@@ -285,7 +286,7 @@ public:
     // From QNonContiguousByteDeviceThreadForwardImpl in HTTP thread:
     void resetUploadDataSlot(bool *r);
     void wantUploadDataSlot(qint64);
-    void sentUploadDataSlot(qint64);
+    void sentUploadDataSlot(qint64, qint64);
 
     // From user's QNonContiguousByteDevice
     void uploadByteDeviceReadyReadSlot();