network API: add support for HTTP multipart messages

This commit adds two new classes, QHttpPart and QHttpMultiPart, and
two new overloads to QNetworkAccessManager:
post(const QNetworkRequest &request, QHttpMultiPart *multiPart) and
put(const QNetworkRequest &request, QHttpMultiPart *multiPart).

With those classes, it is possible to do a HTTP POST with a multipart
message in a memory-saving way: The data from the parts is not copied
when read from a file or another QIODevice.

Reviewed-by: Markus Goetz
Task-number: QTBUG-6222

1 files changed, 343 insertions, 0 deletions

diff --git a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
index 48b3ecc8e7..6ed8f16a08 100644
--- a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
+++ b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp
@@ -60,6 +60,8 @@
 #include <QtNetwork/qnetworkrequest.h>
 #include <QtNetwork/qnetworkreply.h>
 #include <QtNetwork/qnetworkcookie.h>
+#include <QtNetwork/QHttpPart>
+#include <QtNetwork/QHttpMultiPart>
 #ifndef QT_NO_OPENSSL
 #include <QtNetwork/qsslerror.h>
 #include <QtNetwork/qsslconfiguration.h>
@@ -83,6 +85,8 @@ Q_DECLARE_METATYPE(QNetworkProxyQuery)
 Q_DECLARE_METATYPE(QList<QNetworkProxy>)
 Q_DECLARE_METATYPE(QNetworkReply::NetworkError)
 Q_DECLARE_METATYPE(QBuffer*)
+Q_DECLARE_METATYPE(QHttpMultiPart *)
+Q_DECLARE_METATYPE(QList<QFile*>) // for multiparts
 
 class QNetworkReplyPtr: public QSharedPointer<QNetworkReply>
 {
@@ -139,6 +143,8 @@ public:
     ~tst_QNetworkReply();
     QString runSimpleRequest(QNetworkAccessManager::Operation op, const QNetworkRequest &request,
                              QNetworkReplyPtr &reply, const QByteArray &data = QByteArray());
+    QString runMultipartRequest(const QNetworkRequest &request, QNetworkReplyPtr &reply,
+                                    QHttpMultiPart *multiPart, const QByteArray &verb);
 
     QString runCustomRequest(const QNetworkRequest &request, QNetworkReplyPtr &reply,
                              const QByteArray &verb, QIODevice *data);
@@ -185,10 +191,14 @@ private Q_SLOTS:
     void putToHttp();
     void putToHttpSynchronous_data();
     void putToHttpSynchronous();
+    void putToHttpMultipart_data();
+    void putToHttpMultipart();
     void postToHttp_data();
     void postToHttp();
     void postToHttpSynchronous_data();
     void postToHttpSynchronous();
+    void postToHttpMultipart_data();
+    void postToHttpMultipart();
     void deleteFromHttp_data();
     void deleteFromHttp();
     void putGetDeleteGetFromHttp_data();
@@ -1092,6 +1102,38 @@ void tst_QNetworkReply::storeSslConfiguration()
 }
 #endif
 
+QString tst_QNetworkReply::runMultipartRequest(const QNetworkRequest &request,
+                                                   QNetworkReplyPtr &reply,
+                                                   QHttpMultiPart *multiPart,
+                                                   const QByteArray &verb)
+{
+    if (verb == "POST")
+        reply = manager.post(request, multiPart);
+    else
+        reply = manager.put(request, multiPart);
+
+    // the code below is copied from tst_QNetworkReply::runSimpleRequest, see below
+    reply->setParent(this);
+    connect(reply, SIGNAL(finished()), SLOT(finished()));
+    connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(gotError()));
+    multiPart->setParent(reply);
+
+    returnCode = Timeout;
+    loop = new QEventLoop;
+    QTimer::singleShot(25000, loop, SLOT(quit()));
+    int code = returnCode == Timeout ? loop->exec() : returnCode;
+    delete loop;
+    loop = 0;
+
+    switch (code) {
+    case Failure:
+        return "Request failed: " + reply->errorString();
+    case Timeout:
+        return "Network timeout";
+    }
+    return QString();
+}
+
 QString tst_QNetworkReply::runSimpleRequest(QNetworkAccessManager::Operation op,
                                             const QNetworkRequest &request,
                                             QNetworkReplyPtr &reply,
@@ -1837,6 +1879,307 @@ void tst_QNetworkReply::postToHttpSynchronous()
     QCOMPARE(uploadedData, md5sum.toHex());
 }
 
+void tst_QNetworkReply::postToHttpMultipart_data()
+{
+    QTest::addColumn<QUrl>("url");
+    QTest::addColumn<QHttpMultiPart *>("multiPart");
+    QTest::addColumn<QByteArray>("expectedReplyData");
+    QTest::addColumn<QByteArray>("contentType");
+
+    QUrl url("http://" + QtNetworkSettings::serverName() + "/qtest/cgi-bin/multipart.cgi");
+    QByteArray expectedData;
+
+
+    // empty parts
+
+    QHttpMultiPart *emptyMultiPart = new QHttpMultiPart;
+    QTest::newRow("empty") << url << emptyMultiPart << expectedData << QByteArray("mixed");
+
+    QHttpMultiPart *emptyRelatedMultiPart = new QHttpMultiPart;
+    emptyRelatedMultiPart->setContentType(QHttpMultiPart::RelatedType);
+    QTest::newRow("empty-related") << url << emptyRelatedMultiPart << expectedData << QByteArray("related");
+
+    QHttpMultiPart *emptyAlternativeMultiPart = new QHttpMultiPart;
+    emptyAlternativeMultiPart->setContentType(QHttpMultiPart::AlternativeType);
+    QTest::newRow("empty-alternative") << url << emptyAlternativeMultiPart << expectedData << QByteArray("alternative");
+
+
+    // text-only parts
+
+    QHttpPart textPart;
+    textPart.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("text/plain"));
+    textPart.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"text\""));
+    textPart.setBody("7 bytes");
+    QHttpMultiPart *multiPart1 = new QHttpMultiPart;
+    multiPart1->setContentType(QHttpMultiPart::FormDataType);
+    multiPart1->append(textPart);
+    expectedData = "key: text, value: 7 bytes\n";
+    QTest::newRow("text") << url << multiPart1 << expectedData << QByteArray("form-data");
+
+    QHttpMultiPart *customMultiPart = new QHttpMultiPart;
+    customMultiPart->append(textPart);
+    expectedData = "header: Content-Type, value: 'text/plain'\n"
+                   "header: Content-Disposition, value: 'form-data; name=\"text\"'\n"
+                   "content: 7 bytes\n"
+                   "\n";
+    QTest::newRow("text-custom") << url << customMultiPart << expectedData << QByteArray("custom");
+
+    QHttpPart textPart2;
+    textPart2.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("text/plain"));
+    textPart2.setRawHeader("myRawHeader", "myValue");
+    textPart2.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"text2\""));
+    textPart2.setBody("some more bytes");
+    textPart2.setBodyDevice((QIODevice *) 1); // test whether setting and unsetting of the device works
+    textPart2.setBodyDevice(0);
+    QHttpMultiPart *multiPart2 = new QHttpMultiPart;
+    multiPart2->setContentType(QHttpMultiPart::FormDataType);
+    multiPart2->append(textPart);
+    multiPart2->append(textPart2);
+    expectedData = "key: text2, value: some more bytes\n"
+                   "key: text, value: 7 bytes\n";
+    QTest::newRow("text-text") << url << multiPart2 << expectedData << QByteArray("form-data");
+
+
+    QHttpPart textPart3;
+    textPart3.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("text/plain"));
+    textPart3.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"text3\""));
+    textPart3.setRawHeader("Content-Location", "http://my.test.location.tld");
+    textPart3.setBody("even more bytes");
+    QHttpMultiPart *multiPart3 = new QHttpMultiPart;
+    multiPart3->setContentType(QHttpMultiPart::AlternativeType);
+    multiPart3->append(textPart);
+    multiPart3->append(textPart2);
+    multiPart3->append(textPart3);
+    expectedData = "header: Content-Type, value: 'text/plain'\n"
+                   "header: Content-Disposition, value: 'form-data; name=\"text\"'\n"
+                   "content: 7 bytes\n"
+                   "\n"
+                   "header: Content-Type, value: 'text/plain'\n"
+                   "header: myRawHeader, value: 'myValue'\n"
+                   "header: Content-Disposition, value: 'form-data; name=\"text2\"'\n"
+                   "content: some more bytes\n"
+                   "\n"
+                   "header: Content-Type, value: 'text/plain'\n"
+                   "header: Content-Disposition, value: 'form-data; name=\"text3\"'\n"
+                   "header: Content-Location, value: 'http://my.test.location.tld'\n"
+                   "content: even more bytes\n\n";
+    QTest::newRow("text-text-text") << url << multiPart3 << expectedData << QByteArray("alternative");
+
+
+
+    // text and image parts
+
+    QHttpPart imagePart11;
+    imagePart11.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart11.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage\""));
+    imagePart11.setRawHeader("Content-Location", "http://my.test.location.tld");
+    imagePart11.setRawHeader("Content-ID", "my@id.tld");
+    QFile *file11 = new QFile(SRCDIR "/image1.jpg");
+    file11->open(QIODevice::ReadOnly);
+    imagePart11.setBodyDevice(file11);
+    QHttpMultiPart *imageMultiPart1 = new QHttpMultiPart(QHttpMultiPart::FormDataType);
+    imageMultiPart1->append(imagePart11);
+    file11->setParent(imageMultiPart1);
+    expectedData = "key: testImage, value: 87ef3bb319b004ba9e5e9c9fa713776e\n"; // md5 sum of file
+    QTest::newRow("image") << url << imageMultiPart1 << expectedData << QByteArray("form-data");
+
+    QHttpPart imagePart21;
+    imagePart21.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart21.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage1\""));
+    imagePart21.setRawHeader("Content-Location", "http://my.test.location.tld");
+    imagePart21.setRawHeader("Content-ID", "my@id.tld");
+    QFile *file21 = new QFile(SRCDIR "/image1.jpg");
+    file21->open(QIODevice::ReadOnly);
+    imagePart21.setBodyDevice(file21);
+    QHttpMultiPart *imageMultiPart2 = new QHttpMultiPart();
+    imageMultiPart2->setContentType(QHttpMultiPart::FormDataType);
+    imageMultiPart2->append(textPart);
+    imageMultiPart2->append(imagePart21);
+    file21->setParent(imageMultiPart2);
+    QHttpPart imagePart22;
+    imagePart22.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart22.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage2\""));
+    QFile *file22 = new QFile(SRCDIR "/image2.jpg");
+    file22->open(QIODevice::ReadOnly);
+    imagePart22.setBodyDevice(file22);
+    imageMultiPart2->append(imagePart22);
+    file22->setParent(imageMultiPart2);
+    expectedData = "key: testImage1, value: 87ef3bb319b004ba9e5e9c9fa713776e\n"
+                   "key: text, value: 7 bytes\n"
+                   "key: testImage2, value: 483761b893f7fb1bd2414344cd1f3dfb\n";
+    QTest::newRow("text-image-image") << url << imageMultiPart2 << expectedData << QByteArray("form-data");
+
+
+    QHttpPart imagePart31;
+    imagePart31.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart31.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage1\""));
+    imagePart31.setRawHeader("Content-Location", "http://my.test.location.tld");
+    imagePart31.setRawHeader("Content-ID", "my@id.tld");
+    QFile *file31 = new QFile(SRCDIR "/image1.jpg");
+    file31->open(QIODevice::ReadOnly);
+    imagePart31.setBodyDevice(file31);
+    QHttpMultiPart *imageMultiPart3 = new QHttpMultiPart(QHttpMultiPart::FormDataType);
+    imageMultiPart3->append(imagePart31);
+    file31->setParent(imageMultiPart3);
+    QHttpPart imagePart32;
+    imagePart32.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart32.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage2\""));
+    QFile *file32 = new QFile(SRCDIR "/image2.jpg");
+    file32->open(QIODevice::ReadOnly);
+    imagePart32.setBodyDevice(file31); // check that resetting works
+    imagePart32.setBodyDevice(file32);
+    imageMultiPart3->append(imagePart32);
+    file32->setParent(imageMultiPart3);
+    QHttpPart imagePart33;
+    imagePart33.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart33.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage3\""));
+    QFile *file33 = new QFile(SRCDIR "/image3.jpg");
+    file33->open(QIODevice::ReadOnly);
+    imagePart33.setBodyDevice(file33);
+    imageMultiPart3->append(imagePart33);
+    file33->setParent(imageMultiPart3);
+    expectedData = "key: testImage1, value: 87ef3bb319b004ba9e5e9c9fa713776e\n"
+                   "key: testImage2, value: 483761b893f7fb1bd2414344cd1f3dfb\n"
+                   "key: testImage3, value: ab0eb6fd4fcf8b4436254870b4513033\n";
+    QTest::newRow("3-images") << url << imageMultiPart3 << expectedData << QByteArray("form-data");
+
+
+    // note: nesting multiparts is not working currently; for that, the outputDevice would need to be public
+
+//    QHttpPart imagePart41;
+//    imagePart41.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+//    QFile *file41 = new QFile(SRCDIR "/image1.jpg");
+//    file41->open(QIODevice::ReadOnly);
+//    imagePart41.setBodyDevice(file41);
+//
+//    QHttpMultiPart *innerMultiPart = new QHttpMultiPart();
+//    innerMultiPart->setContentType(QHttpMultiPart::FormDataType);
+//    textPart.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant());
+//    innerMultiPart->append(textPart);
+//    innerMultiPart->append(imagePart41);
+//    textPart2.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant());
+//    innerMultiPart->append(textPart2);
+//
+//    QHttpPart nestedPart;
+//    nestedPart.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"nestedMessage"));
+//    nestedPart.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("multipart/alternative; boundary=\"" + innerMultiPart->boundary() + "\""));
+//    innerMultiPart->outputDevice()->open(QIODevice::ReadOnly);
+//    nestedPart.setBodyDevice(innerMultiPart->outputDevice());
+//
+//    QHttpMultiPart *outerMultiPart = new QHttpMultiPart;
+//    outerMultiPart->setContentType(QHttpMultiPart::FormDataType);
+//    outerMultiPart->append(textPart);
+//    outerMultiPart->append(nestedPart);
+//    outerMultiPart->append(textPart2);
+//    expectedData = "nothing"; // the CGI.pm module running on the test server does not understand nested multiparts
+//    openFiles.clear();
+//    openFiles << file41;
+//    QTest::newRow("nested") << url << outerMultiPart << expectedData << openFiles;
+
+
+    // test setting large chunks of content with a byte array instead of a device (DISCOURAGED because of high memory consumption,
+    // but we need to test that the behavior is correct)
+    QHttpPart imagePart51;
+    imagePart51.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("image/jpeg"));
+    imagePart51.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data; name=\"testImage\""));
+    QFile *file51 = new QFile(SRCDIR "/image1.jpg");
+    file51->open(QIODevice::ReadOnly);
+    QByteArray imageData = file51->readAll();
+    file51->close();
+    delete file51;
+    imagePart51.setBody("7 bytes"); // check that resetting works
+    imagePart51.setBody(imageData);
+    QHttpMultiPart *imageMultiPart5 = new QHttpMultiPart;
+    imageMultiPart5->setContentType(QHttpMultiPart::FormDataType);
+    imageMultiPart5->append(imagePart51);
+    expectedData = "key: testImage, value: 87ef3bb319b004ba9e5e9c9fa713776e\n"; // md5 sum of file
+    QTest::newRow("image-as-content") << url << imageMultiPart5 << expectedData << QByteArray("form-data");
+}
+
+void tst_QNetworkReply::postToHttpMultipart()
+{
+    QFETCH(QUrl, url);
+
+    static QSet<QByteArray> boundaries;
+
+    QNetworkRequest request(url);
+    QNetworkReplyPtr reply;
+
+    QFETCH(QHttpMultiPart *, multiPart);
+    QFETCH(QByteArray, expectedReplyData);
+    QFETCH(QByteArray, contentType);
+
+    // hack for testing the setting of the content-type header by hand:
+    if (contentType == "custom") {
+        QByteArray contentType("multipart/custom; boundary=\"" + multiPart->boundary() + "\"");
+        request.setHeader(QNetworkRequest::ContentTypeHeader, contentType);
+    }
+
+    QVERIFY2(! boundaries.contains(multiPart->boundary()), "boundary '" + multiPart->boundary() + "' has been created twice");
+    boundaries.insert(multiPart->boundary());
+
+    RUN_REQUEST(runMultipartRequest(request, reply, multiPart, "POST"));
+    multiPart->deleteLater();
+
+    QCOMPARE(reply->url(), url);
+    QCOMPARE(reply->error(), QNetworkReply::NoError);
+
+    QCOMPARE(reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(), 200); // 200 Ok
+
+    QVERIFY(multiPart->boundary().count() > 20); // check that there is randomness after the "boundary_.oOo._" string
+    QVERIFY(multiPart->boundary().count() < 70);
+    QByteArray replyData = reply->readAll();
+
+    expectedReplyData.prepend("content type: multipart/" + contentType + "; boundary=\"" + multiPart->boundary() + "\"\n");
+//    QEXPECT_FAIL("nested", "the server does not understand nested multipart messages", Continue); // see above
+    QCOMPARE(replyData, expectedReplyData);
+}
+
+void tst_QNetworkReply::putToHttpMultipart_data()
+{
+    postToHttpMultipart_data();
+}
+
+void tst_QNetworkReply::putToHttpMultipart()
+{
+    QSKIP("test server script cannot handle PUT data yet", SkipAll);
+    QFETCH(QUrl, url);
+
+    static QSet<QByteArray> boundaries;
+
+    QNetworkRequest request(url);
+    QNetworkReplyPtr reply;
+
+    QFETCH(QHttpMultiPart *, multiPart);
+    QFETCH(QByteArray, expectedReplyData);
+    QFETCH(QByteArray, contentType);
+
+    // hack for testing the setting of the content-type header by hand:
+    if (contentType == "custom") {
+        QByteArray contentType("multipart/custom; boundary=\"" + multiPart->boundary() + "\"");
+        request.setHeader(QNetworkRequest::ContentTypeHeader, contentType);
+    }
+
+    QVERIFY2(! boundaries.contains(multiPart->boundary()), "boundary '" + multiPart->boundary() + "' has been created twice");
+    boundaries.insert(multiPart->boundary());
+
+    RUN_REQUEST(runMultipartRequest(request, reply, multiPart, "PUT"));
+    multiPart->deleteLater();
+
+    QCOMPARE(reply->url(), url);
+    QCOMPARE(reply->error(), QNetworkReply::NoError);
+
+    QCOMPARE(reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(), 200); // 200 Ok
+
+    QVERIFY(multiPart->boundary().count() > 20); // check that there is randomness after the "boundary_.oOo._" string
+    QVERIFY(multiPart->boundary().count() < 70);
+    QByteArray replyData = reply->readAll();
+
+    expectedReplyData.prepend("content type: multipart/" + contentType + "; boundary=\"" + multiPart->boundary() + "\"\n");
+//    QEXPECT_FAIL("nested", "the server does not understand nested multipart messages", Continue); // see above
+    QCOMPARE(replyData, expectedReplyData);
+}
+
 void tst_QNetworkReply::deleteFromHttp_data()
 {
     QTest::addColumn<QUrl>("url");