added support for BOM(byte order mark) before utf; added utf32

Change-Id: Iefffd320293e3324dcf29d2f2bb284b3cdf76e97
Reviewed-by: Andrew Christian <andrew.christian@nokia.com>

7 files changed, 231 insertions, 26 deletions

diff --git a/src/jsonbuffer.cpp b/src/jsonbuffer.cpp
index 3fe5561..7d54c83 100644
--- a/src/jsonbuffer.cpp
+++ b/src/jsonbuffer.cpp
@@ -51,6 +51,12 @@
 
 QT_BEGIN_NAMESPACE_JSONSTREAM
 
+template <typename T>
+inline bool isjsonws(T c)
+{
+    return c == '\n' || c == ' ' || c == '\t' || c == '\r';
+}
+
 /*!
   \class JsonBuffer
   \brief The JsonBuffer class parses data received into appropriate Json messages
@@ -184,7 +190,6 @@ bool JsonBuffer::scanUtf( int c )
         else if ( c == '}' && mParserDepth > 0 ) {
             mParserDepth -= 1;
             if ( mParserDepth == 0 ) {
-                mParserOffset++;
                 return true;
             }
         }
@@ -242,23 +247,52 @@ bool JsonBuffer::messageAvailable()
         return true;
     }
 
+    if (mBuffer.size() < 4) {
+        // buffer too small for a json message
+        return false;
+    }
+
     if (mFormat == FormatUndefined && mBuffer.size() >= 4) {
         if (strncmp("bson", mBuffer.constData(), 4) == 0)
             mFormat = FormatBSON;
         else if (QJsonDocument::BinaryFormatTag == *((uint *) mBuffer.constData()))
             mFormat = FormatQBJS;
-        else if (mBuffer.at(0) == 0 &&
-                 mBuffer.at(1) != 0 &&
-                 mBuffer.at(2) == 0 &&
-                 mBuffer.at(3) != 0 )
-            mFormat = FormatUTF16BE;
-        else if (mBuffer.at(0) != 0 &&
-                 mBuffer.at(1) == 0 &&
-                 mBuffer.at(2) != 0 &&
-                 mBuffer.at(3) == 0 )
-            mFormat = FormatUTF16LE;
-        else
-            mFormat = FormatUTF8;
+        else {
+            uchar u0 = mBuffer.at(0), u1 = mBuffer.at(1), u2 = mBuffer.at(2), u3 = mBuffer.at(3);
+            // has a BOM?
+            if (u0 == 0xFF && u1 == 0xFE) { // utf-32 le or utf-16 le + BOM
+                mFormat = (u2 == 0 && u3 == 0 ) ? FormatUTF32LE : FormatUTF16LE;
+                mParserOffset++;
+            }
+            else if (u0 == 0xFE && u1 == 0xFF) { // utf16 be + BOM
+                mFormat = FormatUTF16BE;
+                mParserOffset++;
+            }
+            else if (u0 == 0x00 && u1 == 0x00 && u2 == 0xFE && u3 == 0xFF ) { // utf-32 be + BOM
+                mFormat = FormatUTF32BE;
+                mParserOffset++;
+            }
+            else if (u0 == 0xEF && u1 == 0xBB && u2 == 0xBF) { // utf8 + BOM
+                mFormat = FormatUTF8;
+                mParserOffset+=3;
+            }
+            // no BOM
+            else if (u0 == 0 && u1 != 0 && u2 == 0 && u3 != 0 ) {
+                mFormat = FormatUTF16BE;
+            }
+            else if (u0 != 0 && u1 == 0 && u2 != 0 && u3 == 0 ) {
+                mFormat = FormatUTF16LE;
+            }
+            else if (u0 == 0 && u1 == 0 && u2 == 0 && u3 != 0 ) {
+                mFormat = FormatUTF32BE;
+            }
+            else if (u0 != 0 && u1 == 0 && u2 == 0 && u3 == 0 ) {
+                mFormat = FormatUTF32LE;
+            }
+            else {
+                mFormat = FormatUTF8;
+            }
+        }
     }
 
     switch (mFormat) {
@@ -267,27 +301,60 @@ bool JsonBuffer::messageAvailable()
     case FormatUTF8:
         for (  ; mParserOffset < mBuffer.size() ; mParserOffset++ ) {
             char c = mBuffer.at(mParserOffset);
-            if (scanUtf(c)) {
+            if (mMessageAvailable) {
+                if (!isjsonws(c))
+                    break;
+            }
+            else if (scanUtf(c)) {
                 mMessageAvailable = true;
-                return true;
             }
         }
         break;
     case FormatUTF16BE:
         for (  ; 2 * mParserOffset < mBuffer.size() ; mParserOffset++ ) {
             int16_t c = qFromBigEndian(reinterpret_cast<const int16_t *>(mBuffer.constData())[mParserOffset]);
-            if (scanUtf(c)) {
+            if (mMessageAvailable) {
+                if (!isjsonws(c))
+                    break;
+            }
+            else if (scanUtf(c)) {
                 mMessageAvailable = true;
-                return true;
             }
         }
         break;
     case FormatUTF16LE:
         for (  ; 2 * mParserOffset < mBuffer.size() ; mParserOffset++ ) {
             int16_t c = qFromLittleEndian(reinterpret_cast<const int16_t *>(mBuffer.constData())[mParserOffset]);
-            if (scanUtf(c)) {
+            if (mMessageAvailable) {
+                if (!isjsonws(c))
+                    break;
+            }
+            else if (scanUtf(c)) {
+                mMessageAvailable = true;
+            }
+        }
+        break;
+    case FormatUTF32BE:
+        for (  ; 4 * mParserOffset < mBuffer.size() ; mParserOffset++ ) {
+            int32_t c = qFromBigEndian(reinterpret_cast<const int32_t *>(mBuffer.constData())[mParserOffset]);
+            if (mMessageAvailable) {
+                if (!isjsonws(c))
+                    break;
+            }
+            else if (scanUtf(c)) {
+                mMessageAvailable = true;
+            }
+        }
+        break;
+    case FormatUTF32LE:
+        for (  ; 4 * mParserOffset < mBuffer.size() ; mParserOffset++ ) {
+            int32_t c = qFromLittleEndian(reinterpret_cast<const int32_t *>(mBuffer.constData())[mParserOffset]);
+            if (mMessageAvailable) {
+                if (!isjsonws(c))
+                    break;
+            }
+            else if (scanUtf(c)) {
                 mMessageAvailable = true;
-                return true;
             }
         }
         break;
@@ -353,6 +420,26 @@ QJsonObject JsonBuffer::readMessage()
                 resetParser();
             }
             break;
+        case FormatUTF32BE:
+            if (mParserStartOffset >= 0) {
+                QByteArray msg = rawData(mParserStartOffset * 4, 4*(mParserOffset - mParserStartOffset));
+                QString s = QTextCodec::codecForName("UTF-32BE")->toUnicode(msg);
+                obj = QJsonDocument::fromJson(s.toUtf8()).object();
+                // prepare for the next
+                mBuffer.remove(0, mParserOffset*4);
+                resetParser();
+            }
+            break;
+        case FormatUTF32LE:
+            if (mParserStartOffset >= 0) {
+                QByteArray msg = rawData(mParserStartOffset * 4, 4*(mParserOffset - mParserStartOffset));
+                QString s = QTextCodec::codecForName("UTF-32LE")->toUnicode(msg);
+                obj = QJsonDocument::fromJson(s.toUtf8()).object();
+                // prepare for the next
+                mBuffer.remove(0, mParserOffset*4);
+                resetParser();
+            }
+            break;
         case FormatBSON:
             if (mMessageSize > 0) {
                 QByteArray msg = rawData(4, mMessageSize);
diff --git a/src/jsonpipe.cpp b/src/jsonpipe.cpp
index 18c8f46..dd0fa78 100644
--- a/src/jsonpipe.cpp
+++ b/src/jsonpipe.cpp
@@ -250,6 +250,12 @@ bool JsonPipe::send(const QJsonObject& object)
     case FormatUTF16LE:
         d->mOutBuffer.append( QTextCodec::codecForName("UTF-16LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) );
         break;
+    case FormatUTF32BE:
+        d->mOutBuffer.append( QTextCodec::codecForName("UTF-32BE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(4) );
+        break;
+    case FormatUTF32LE:
+        d->mOutBuffer.append( QTextCodec::codecForName("UTF-32LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(4) );
+        break;
     case FormatBSON:
     {
         BsonObject bson(document.toVariant().toMap());
diff --git a/src/jsonstream-global.h b/src/jsonstream-global.h
index 05e6e95..1a02fcf 100644
--- a/src/jsonstream-global.h
+++ b/src/jsonstream-global.h
@@ -67,7 +67,7 @@
 
 QT_BEGIN_NAMESPACE_JSONSTREAM
 
-enum EncodingFormat { FormatUndefined, FormatUTF8, FormatBSON, FormatQBJS, FormatUTF16BE, FormatUTF16LE };
+enum EncodingFormat { FormatUndefined, FormatUTF8, FormatBSON, FormatQBJS, FormatUTF16BE, FormatUTF16LE, FormatUTF32BE, FormatUTF32LE };
 
 QT_END_NAMESPACE_JSONSTREAM
 
diff --git a/src/jsonstream.cpp b/src/jsonstream.cpp
index 750dc51..9a022a8 100644
--- a/src/jsonstream.cpp
+++ b/src/jsonstream.cpp
@@ -233,6 +233,12 @@ bool JsonStream::send(const QJsonObject& object)
     case FormatUTF16LE:
         bRet = sendInternal( QTextCodec::codecForName("UTF-16LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) );  // Chop off BOM
         break;
+    case FormatUTF32BE:
+        bRet = sendInternal( QTextCodec::codecForName("UTF-32BE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(4) );  // Chop off BOM
+        break;
+    case FormatUTF32LE:
+        bRet = sendInternal( QTextCodec::codecForName("UTF-32LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(4) );  // Chop off BOM
+        break;
     case FormatBSON:
     {
         BsonObject bson(document.toVariant().toMap());
diff --git a/tests/auto/jsonbuffer/tst_jsonbuffer.cpp b/tests/auto/jsonbuffer/tst_jsonbuffer.cpp
index 635d184..13acf64 100644
--- a/tests/auto/jsonbuffer/tst_jsonbuffer.cpp
+++ b/tests/auto/jsonbuffer/tst_jsonbuffer.cpp
@@ -55,7 +55,7 @@ private slots:
 };
 
 
-const char *utf8spaces[] = { "{\"a\":123.0}{\"b\":234}",
+const char *utf8spaces[] = { "{\"a\":123.0}{\"b\":234}\t \r\n", // test for a buffer size
                              "{\"a\": 123}   \n{\"b\"   :234  }  ",
                              "{     \"a\"  : 123.0000}   \n { \"b\"   :234  }  ",
                              "\n    {\"a\": 123}   \n{\"b\"   :234  }  ",
@@ -80,6 +80,9 @@ void tst_JsonBuffer::utf8()
 
         QVERIFY(!buf.messageAvailable());
         QVERIFY(buf.format() == FormatUTF8);
+
+        if (0 == i)
+            QVERIFY(buf.size() == 0); // buffer should be empty at the end
     }
 }
 
@@ -95,7 +98,7 @@ PartialData utf8packets[] = { { "{\"a\":123.0", 0, 0 },
                               { "  \n{\"c\"   :  1000.0  }  ", "c", 1000.0 },
                               { "{\"value\":  ", 0, 0 },
                               { " 1.0", 0, 0 },
-                              { " \t\t\t}", "value", 1.0 }
+                              { " \t\t\t}\t \r\n", "value", 1.0 } // test for a buffer size
 };
 
 void tst_JsonBuffer::utf8extend()
@@ -114,6 +117,7 @@ void tst_JsonBuffer::utf8extend()
         QVERIFY(!buf.messageAvailable());
         QVERIFY(buf.format() == FormatUTF8);
     }
+    QVERIFY(buf.size() == 0); // buffer should be empty at the end
 }
 
 QTEST_MAIN(tst_JsonBuffer)
diff --git a/tests/auto/jsonstream/testClient/main.cpp b/tests/auto/jsonstream/testClient/main.cpp
index e23e301..5ba9b65 100644
--- a/tests/auto/jsonstream/testClient/main.cpp
+++ b/tests/auto/jsonstream/testClient/main.cpp
@@ -87,6 +87,10 @@ Container::Container()
         mClient->setFormat(FormatUTF16BE);
     else if (gFormat == "utf16le")
         mClient->setFormat(FormatUTF16LE);
+    else if (gFormat == "utf32be")
+        mClient->setFormat(FormatUTF32BE);
+    else if (gFormat == "utf32le")
+        mClient->setFormat(FormatUTF32LE);
 
     if (!mClient->connectLocal(gSocketname)) {
         qWarning() << "Unable to connect to" << gSocketname;
diff --git a/tests/auto/jsonstream/tst_jsonstream.cpp b/tests/auto/jsonstream/tst_jsonstream.cpp
index e997639..4ec119b 100644
--- a/tests/auto/jsonstream/tst_jsonstream.cpp
+++ b/tests/auto/jsonstream/tst_jsonstream.cpp
@@ -170,6 +170,16 @@ private:
 };
 
 /****************************/
+class TestJsonStream : public JsonStream
+{
+public:
+    TestJsonStream(QIODevice *device)
+        : JsonStream(device) {}
+
+    bool sendRaw(const QByteArray& byteArray) { return sendInternal(byteArray); }
+};
+
+/****************************/
 class BasicServer : public QObject {
     Q_OBJECT
 
@@ -194,6 +204,10 @@ public:
         return stream ? stream->send(message) : false;
     }
 
+    bool sendRaw(const QByteArray& buffer) {
+        return stream ? stream->sendRaw(buffer) : false;
+    }
+
     void waitDisconnect(int timeout=5000) {
         QTime stopWatch;
         stopWatch.start();
@@ -215,7 +229,7 @@ private slots:
     void handleConnection() {
         socket = server->nextPendingConnection();
         QVERIFY(socket);
-        stream = new JsonStream(socket);
+        stream = new TestJsonStream(socket);
         stream->setParent(socket);
         if (readBufferSize > 0)
             stream->setReadBufferSize(readBufferSize);
@@ -255,7 +269,7 @@ signals:
 private:
     QLocalServer *server;
     QLocalSocket *socket;
-    JsonStream   *stream;
+    TestJsonStream   *stream;
     qint64        readBufferSize;
     bool          mHandleReadBufOverflow;
 public:
@@ -277,6 +291,7 @@ private slots:
     void authRangeTest();
     void authRangeFail();
     void formatTest();
+    void BOMformatTest();
     void schemaTest();
     void pipeTest();
     void pipeFormatTest();
@@ -397,7 +412,7 @@ void tst_JsonStream::formatTest()
 {
     QString socketname = "/tmp/tst_socket";
 
-    QStringList formats = QStringList() << "qbjs" << "bson" << "utf8" << "utf16be" << "utf16le";
+    QStringList formats = QStringList() << "qbjs" << "bson" << "utf8" << "utf16be" << "utf16le" << "utf32be" << "utf32le";
 
     foreach (const QString& format, formats) {
         BasicServer server(socketname);
@@ -426,6 +441,10 @@ void tst_JsonStream::formatTest()
             QVERIFY(server.format() == FormatUTF16BE);
         else if (format == "utf16le")
             QVERIFY(server.format() == FormatUTF16LE);
+        else if (format == "utf32be")
+            QVERIFY(server.format() == FormatUTF32BE);
+        else if (format == "utf32le")
+            QVERIFY(server.format() == FormatUTF32LE);
         else
             QFAIL("Unrecognized format");
 
@@ -453,6 +472,85 @@ void tst_JsonStream::formatTest()
     }
 }
 
+void tst_JsonStream::BOMformatTest()
+{
+    QString socketname = "/tmp/tst_socket";
+
+    QStringList formats = QStringList() << "utf8" << "utf16be" << "utf16le" << "utf32be" << "utf32le";
+
+    foreach (const QString& format, formats) {
+        BasicServer server(socketname);
+        QSignalSpy spy(&server, SIGNAL(messageReceived(const QJsonObject&)));
+        QTime stopWatch;
+
+        Child child("testClient/testClient",
+                    QStringList() << "-socket" << socketname << "-format" << format);
+
+        stopWatch.start();
+        forever {
+            QTestEventLoop::instance().enterLoop(1);
+            if (stopWatch.elapsed() >= 5000)
+                QFAIL("Timed out");
+            if (spy.count())
+                break;
+        }
+
+        if (format == "utf8")
+            QVERIFY(server.format() == FormatUTF8);
+        else if (format == "utf16be")
+            QVERIFY(server.format() == FormatUTF16BE);
+        else if (format == "utf16le")
+            QVERIFY(server.format() == FormatUTF16LE);
+        else if (format == "utf32be")
+            QVERIFY(server.format() == FormatUTF32BE);
+        else if (format == "utf32le")
+            QVERIFY(server.format() == FormatUTF32LE);
+        else
+            QFAIL("Unrecognized format");
+
+        QJsonObject msg = qvariant_cast<QJsonObject>(spy.last().at(0));
+        QVERIFY(msg.value("text").isString() && msg.value("text").toString() == QLatin1String("Standard text"));
+        QVERIFY(msg.value("int").isDouble() && msg.value("int").toDouble() == 100);
+        QVERIFY(msg.value("float").isDouble() && msg.value("float").toDouble() == 100.0);
+        QVERIFY(msg.value("true").isBool() && msg.value("true").toBool() == true);
+        QVERIFY(msg.value("false").isBool() && msg.value("false").toBool() == false);
+        QVERIFY(msg.value("array").isArray());
+        QJsonArray array = msg.value("array").toArray();
+        QVERIFY(array.size() == 3);
+        QVERIFY(array.at(0).toString() == "one");
+        QVERIFY(array.at(1).toString() == "two");
+        QVERIFY(array.at(2).toString() == "three");
+        QVERIFY(msg.value("object").isObject());
+        QJsonObject object = msg.value("object").toObject();
+        QVERIFY(object.value("item1").toString() == "This is item 1");
+        QVERIFY(object.value("item2").toString() == "This is item 2");
+
+        msg.insert("command", QLatin1String("exit"));
+
+        QJsonDocument document(msg);
+        QString str(QString::fromUtf8(document.toJson()));
+        const char *codec;
+        if (format == "utf8")
+            codec = "UTF-8";
+        else if (format == "utf16be")
+            codec = "UTF-16BE";
+        else if (format == "utf16le")
+            codec = "UTF-16LE";
+        else if (format == "utf32be")
+            codec = "UTF-32BE";
+        else if (format == "utf32le")
+            codec = "UTF-32LE";
+        else
+            QFAIL("Unrecognized format");
+
+        // send data with BOM prepended
+        QTextCodec::ConverterState state(QTextCodec::DefaultConversion);
+        QVERIFY(server.sendRaw(QTextCodec::codecForName(codec)->fromUnicode(str.constData(), str.length(), &state))); // BOM added
+        server.waitDisconnect();
+        child.waitForFinished();
+    }
+}
+
 void tst_JsonStream::bufferSizeTest()
 {
     QString socketname = "/tmp/tst_socket";
@@ -649,7 +747,7 @@ void tst_JsonStream::pipeTest()
 
 void tst_JsonStream::pipeFormatTest()
 {
-    QList<EncodingFormat> formats = QList<EncodingFormat>() << FormatUTF8 << FormatBSON << FormatQBJS << FormatUTF16BE << FormatUTF16LE;
+    QList<EncodingFormat> formats = QList<EncodingFormat>() << FormatUTF8 << FormatBSON << FormatQBJS << FormatUTF16BE << FormatUTF16LE << FormatUTF32BE << FormatUTF32LE;
 
     foreach (EncodingFormat format, formats) {
         Pipes pipes;