Added UTF16-BE and UTF16-LE support

Change-Id: I4322a4b89ef9e8003c3b1951034aee8db64b5ffd
Reviewed-by: Andrew Christian <andrew.christian@nokia.com>

7 files changed, 119 insertions, 40 deletions

diff --git a/src/jsonbuffer.cpp b/src/jsonbuffer.cpp
index 4382713..fde637f 100644
--- a/src/jsonbuffer.cpp
+++ b/src/jsonbuffer.cpp
@@ -42,6 +42,7 @@
 #include <QDebug>
 #include <QtEndian>
 #include <QJsonDocument>
+#include <QTextCodec>
 
 #include "jsonbuffer_p.h"
 #include "qjsondocument.h"
@@ -136,6 +137,52 @@ void JsonBuffer::clear()
   \internal
 */
 
+bool JsonBuffer::scanUtf( int c )
+{
+    switch (mParserState) {
+    case ParseNormal:
+        if ( c == '{' ) {
+            if ( mParserDepth == 0 )
+                mParserStartOffset = mParserOffset;
+            mParserDepth += 1;
+        }
+        else if ( c == '}' && mParserDepth > 0 ) {
+            mParserDepth -= 1;
+            if ( mParserDepth == 0 ) {
+                mParserOffset++;
+                return true;
+            }
+        }
+        else if ( c == '"' ) {
+            mParserState = ParseInString;
+        }
+        break;
+    case ParseInString:
+        if ( c == '"' ) {
+            mParserState = ParseNormal;
+        } else if ( c == '\\' ) {
+            mParserState = ParseInBackslash;
+        }
+        break;
+    case ParseInBackslash:
+        mParserState = ParseInString;
+        break;
+    }
+    return false;
+}
+
+void JsonBuffer::resetParser()
+{
+    mParserState  = ParseNormal;
+    mParserDepth  = 0;
+    mParserOffset = 0;
+    mParserStartOffset = -1;
+}
+
+/*!
+  \internal
+*/
+
 void JsonBuffer::processMessages()
 {
     if (mFormat == FormatUndefined && mBuffer.size() >= 4) {
@@ -143,6 +190,16 @@ void JsonBuffer::processMessages()
             mFormat = FormatBSON;
         else if (QJsonDocument::BinaryFormatTag == *((uint *) mBuffer.data()))
             mFormat = FormatQBJS;
+        else if (mBuffer.at(0) == 0 &&
+                 mBuffer.at(1) != 0 &&
+                 mBuffer.at(2) == 0 &&
+                 mBuffer.at(3) != 0 )
+            mFormat = FormatUTF16BE;
+        else if (mBuffer.at(0) != 0 &&
+                 mBuffer.at(1) == 0 &&
+                 mBuffer.at(2) != 0 &&
+                 mBuffer.at(3) == 0 )
+            mFormat = FormatUTF16LE;
         else
             mFormat = FormatUTF8;
     }
@@ -153,43 +210,41 @@ void JsonBuffer::processMessages()
     case FormatUTF8:
         for (  ; mParserOffset < mBuffer.size() ; mParserOffset++ ) {
             char c = mBuffer.at(mParserOffset);
-            // qDebug() << "Parsing: " << (int) c << c;
-            switch (mParserState) {
-            case ParseNormal:
-                if ( c == '{' ) {
-                    if ( mParserDepth == 0 )
-                        mParserStartOffset = mParserOffset;
-                    mParserDepth += 1;
-                }
-                else if ( c == '}' && mParserDepth > 0 ) {
-                    mParserDepth -= 1;
-                    if ( mParserDepth == 0 ) {
-                        mParserOffset++;
-                        QByteArray msg = mBuffer.mid(mParserStartOffset, mParserOffset - mParserStartOffset);
-                        QJsonObject obj = QJsonDocument::fromJson(msg).object();
-                        if (!obj.isEmpty())
-                            emit objectReceived(obj);
-                        mBuffer = mBuffer.mid(mParserOffset);
-                        mParserState  = ParseNormal;
-                        mParserDepth  = 0;
-                        mParserOffset = 0;
-                        mParserStartOffset = -1;
-                    }
-                }
-                else if ( c == '"' ) {
-                    mParserState = ParseInString;
-                }
-                break;
-            case ParseInString:
-                if ( c == '"' ) {
-                    mParserState = ParseNormal;
-                } else if ( c == '\\' ) {
-                    mParserState = ParseInBackslash;
-                }
-                break;
-            case ParseInBackslash:
-                mParserState = ParseInString;
-                break;
+            if (scanUtf(c)) {
+                QByteArray msg = mBuffer.mid(mParserStartOffset, mParserOffset - mParserStartOffset);
+                QJsonObject obj = QJsonDocument::fromJson(msg).object();
+                if (!obj.isEmpty())
+                    emit objectReceived(obj);
+                mBuffer = mBuffer.mid(mParserOffset);
+                resetParser();
+            }
+        }
+        break;
+    case FormatUTF16BE:
+        for (  ; 2 * mParserOffset < mBuffer.size() ; mParserOffset++ ) {
+            int16_t c = qFromBigEndian(reinterpret_cast<const int16_t *>(mBuffer.constData())[mParserOffset]);
+            if (scanUtf(c)) {
+                QByteArray msg = mBuffer.mid(mParserStartOffset * 2, 2*(mParserOffset - mParserStartOffset));
+                QString s = QTextCodec::codecForName("UTF-16BE")->toUnicode(msg);
+                QJsonObject obj = QJsonDocument::fromJson(s.toUtf8()).object();
+                if (!obj.isEmpty())
+                    emit objectReceived(obj);
+                mBuffer = mBuffer.mid(mParserOffset*2);
+                resetParser();
+            }
+        }
+        break;
+    case FormatUTF16LE:
+        for (  ; 2 * mParserOffset < mBuffer.size() ; mParserOffset++ ) {
+            int16_t c = qFromLittleEndian(reinterpret_cast<const int16_t *>(mBuffer.constData())[mParserOffset]);
+            if (scanUtf(c)) {
+                QByteArray msg = mBuffer.mid(mParserStartOffset * 2, 2*(mParserOffset - mParserStartOffset));
+                QString s = QTextCodec::codecForName("UTF-16LE")->toUnicode(msg);
+                QJsonObject obj = QJsonDocument::fromJson(s.toUtf8()).object();
+                if (!obj.isEmpty())
+                    emit objectReceived(obj);
+                mBuffer = mBuffer.mid(mParserOffset*2);
+                resetParser();
             }
         }
         break;
diff --git a/src/jsonbuffer_p.h b/src/jsonbuffer_p.h
index 3d72be9..9bce2ce 100644
--- a/src/jsonbuffer_p.h
+++ b/src/jsonbuffer_p.h
@@ -67,6 +67,8 @@ signals:
 
 private:
     void processMessages();
+    bool scanUtf(int c);
+    void resetParser();
 
 private:
     enum UTF8ParsingState { ParseNormal, ParseInString, ParseInBackslash };
diff --git a/src/jsonpipe.cpp b/src/jsonpipe.cpp
index 58fb15c..981a25e 100644
--- a/src/jsonpipe.cpp
+++ b/src/jsonpipe.cpp
@@ -45,6 +45,7 @@
 #include <QElapsedTimer>
 #include <qjsondocument.h>
 #include <qjsonobject.h>
+#include <QTextCodec>
 
 #include <sys/select.h>
 #include <stdio.h>
@@ -244,6 +245,12 @@ bool JsonPipe::send(const QJsonObject& object)
     case FormatUTF8:
         d->mOutBuffer.append(document.toJson());
         break;
+    case FormatUTF16BE:
+        d->mOutBuffer.append( QTextCodec::codecForName("UTF-16BE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) );
+        break;
+    case FormatUTF16LE:
+        d->mOutBuffer.append( QTextCodec::codecForName("UTF-16LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) );
+        break;
     case FormatBSON:
     {
         BsonObject bson(document.toVariant().toMap());
diff --git a/src/jsonstream-global.h b/src/jsonstream-global.h
index 3edff30..05e6e95 100644
--- a/src/jsonstream-global.h
+++ b/src/jsonstream-global.h
@@ -67,7 +67,7 @@
 
 QT_BEGIN_NAMESPACE_JSONSTREAM
 
-enum EncodingFormat { FormatUndefined, FormatUTF8, FormatBSON, FormatQBJS };
+enum EncodingFormat { FormatUndefined, FormatUTF8, FormatBSON, FormatQBJS, FormatUTF16BE, FormatUTF16LE };
 
 QT_END_NAMESPACE_JSONSTREAM
 
diff --git a/src/jsonstream.cpp b/src/jsonstream.cpp
index 0fe966a..348eb2d 100644
--- a/src/jsonstream.cpp
+++ b/src/jsonstream.cpp
@@ -45,6 +45,7 @@
 #include <QAbstractSocket>
 #include <QtEndian>
 #include <qjsondocument.h>
+#include <QTextCodec>
 
 #include "jsonstream.h"
 #include "jsonbuffer_p.h"
@@ -187,6 +188,12 @@ void JsonStream::send(const QJsonObject& object)
     case FormatUTF8:
         sendInternal( document.toJson() );
         break;
+    case FormatUTF16BE:
+        sendInternal( QTextCodec::codecForName("UTF-16BE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) );  // Chop off BOM
+        break;
+    case FormatUTF16LE:
+        sendInternal( QTextCodec::codecForName("UTF-16LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) );  // Chop off BOM
+        break;
     case FormatBSON:
     {
         BsonObject bson(document.toVariant().toMap());
diff --git a/tests/auto/jsonstream/testClient/main.cpp b/tests/auto/jsonstream/testClient/main.cpp
index a01fa35..79e9057 100644
--- a/tests/auto/jsonstream/testClient/main.cpp
+++ b/tests/auto/jsonstream/testClient/main.cpp
@@ -83,6 +83,10 @@ Container::Container()
         mClient->setFormat(FormatBSON);
     else if (gFormat == "utf" || gFormat == "utf8")
         mClient->setFormat(FormatUTF8);
+    else if (gFormat == "utf16be")
+        mClient->setFormat(FormatUTF16BE);
+    else if (gFormat == "utf16le")
+        mClient->setFormat(FormatUTF16LE);
 
     if (!mClient->connectLocal(gSocketname)) {
         qWarning() << "Unable to connect to" << gSocketname;
diff --git a/tests/auto/jsonstream/tst_jsonstream.cpp b/tests/auto/jsonstream/tst_jsonstream.cpp
index 39b20ff..14f3b08 100644
--- a/tests/auto/jsonstream/tst_jsonstream.cpp
+++ b/tests/auto/jsonstream/tst_jsonstream.cpp
@@ -367,7 +367,7 @@ void tst_JsonStream::formatTest()
 {
     QString socketname = "/tmp/tst_socket";
 
-    QStringList formats = QStringList() << "qbjs" << "bson" << "utf8";
+    QStringList formats = QStringList() << "qbjs" << "bson" << "utf8" << "utf16be" << "utf16le";
 
     foreach (const QString& format, formats) {
         BasicServer server(socketname);
@@ -392,6 +392,10 @@ void tst_JsonStream::formatTest()
             QVERIFY(server.format() == FormatBSON);
         else if (format == "utf8")
             QVERIFY(server.format() == FormatUTF8);
+        else if (format == "utf16be")
+            QVERIFY(server.format() == FormatUTF16BE);
+        else if (format == "utf16le")
+            QVERIFY(server.format() == FormatUTF16LE);
         else
             QFAIL("Unrecognized format");
 
@@ -533,7 +537,7 @@ void tst_JsonStream::pipeTest()
 
 void tst_JsonStream::pipeFormatTest()
 {
-    QList<EncodingFormat> formats = QList<EncodingFormat>() << FormatUTF8 << FormatBSON << FormatQBJS;
+    QList<EncodingFormat> formats = QList<EncodingFormat>() << FormatUTF8 << FormatBSON << FormatQBJS << FormatUTF16BE << FormatUTF16LE;
 
     foreach (EncodingFormat format, formats) {
         Pipes pipes;