From 89a2d0ac0713a20506439c371bd29d25d82fb741 Mon Sep 17 00:00:00 2001 From: Andrew Christian Date: Mon, 19 Mar 2012 08:55:25 -0400 Subject: Added UTF16-BE and UTF16-LE support Change-Id: I4322a4b89ef9e8003c3b1951034aee8db64b5ffd Reviewed-by: Andrew Christian --- src/jsonbuffer.cpp | 129 +++++++++++++++++++++--------- src/jsonbuffer_p.h | 2 + src/jsonpipe.cpp | 7 ++ src/jsonstream-global.h | 2 +- src/jsonstream.cpp | 7 ++ tests/auto/jsonstream/testClient/main.cpp | 4 + tests/auto/jsonstream/tst_jsonstream.cpp | 8 +- 7 files changed, 119 insertions(+), 40 deletions(-) diff --git a/src/jsonbuffer.cpp b/src/jsonbuffer.cpp index 4382713..fde637f 100644 --- a/src/jsonbuffer.cpp +++ b/src/jsonbuffer.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include "jsonbuffer_p.h" #include "qjsondocument.h" @@ -136,6 +137,52 @@ void JsonBuffer::clear() \internal */ +bool JsonBuffer::scanUtf( int c ) +{ + switch (mParserState) { + case ParseNormal: + if ( c == '{' ) { + if ( mParserDepth == 0 ) + mParserStartOffset = mParserOffset; + mParserDepth += 1; + } + else if ( c == '}' && mParserDepth > 0 ) { + mParserDepth -= 1; + if ( mParserDepth == 0 ) { + mParserOffset++; + return true; + } + } + else if ( c == '"' ) { + mParserState = ParseInString; + } + break; + case ParseInString: + if ( c == '"' ) { + mParserState = ParseNormal; + } else if ( c == '\\' ) { + mParserState = ParseInBackslash; + } + break; + case ParseInBackslash: + mParserState = ParseInString; + break; + } + return false; +} + +void JsonBuffer::resetParser() +{ + mParserState = ParseNormal; + mParserDepth = 0; + mParserOffset = 0; + mParserStartOffset = -1; +} + +/*! + \internal +*/ + void JsonBuffer::processMessages() { if (mFormat == FormatUndefined && mBuffer.size() >= 4) { @@ -143,6 +190,16 @@ void JsonBuffer::processMessages() mFormat = FormatBSON; else if (QJsonDocument::BinaryFormatTag == *((uint *) mBuffer.data())) mFormat = FormatQBJS; + else if (mBuffer.at(0) == 0 && + mBuffer.at(1) != 0 && + mBuffer.at(2) == 0 && + mBuffer.at(3) != 0 ) + mFormat = FormatUTF16BE; + else if (mBuffer.at(0) != 0 && + mBuffer.at(1) == 0 && + mBuffer.at(2) != 0 && + mBuffer.at(3) == 0 ) + mFormat = FormatUTF16LE; else mFormat = FormatUTF8; } @@ -153,43 +210,41 @@ void JsonBuffer::processMessages() case FormatUTF8: for ( ; mParserOffset < mBuffer.size() ; mParserOffset++ ) { char c = mBuffer.at(mParserOffset); - // qDebug() << "Parsing: " << (int) c << c; - switch (mParserState) { - case ParseNormal: - if ( c == '{' ) { - if ( mParserDepth == 0 ) - mParserStartOffset = mParserOffset; - mParserDepth += 1; - } - else if ( c == '}' && mParserDepth > 0 ) { - mParserDepth -= 1; - if ( mParserDepth == 0 ) { - mParserOffset++; - QByteArray msg = mBuffer.mid(mParserStartOffset, mParserOffset - mParserStartOffset); - QJsonObject obj = QJsonDocument::fromJson(msg).object(); - if (!obj.isEmpty()) - emit objectReceived(obj); - mBuffer = mBuffer.mid(mParserOffset); - mParserState = ParseNormal; - mParserDepth = 0; - mParserOffset = 0; - mParserStartOffset = -1; - } - } - else if ( c == '"' ) { - mParserState = ParseInString; - } - break; - case ParseInString: - if ( c == '"' ) { - mParserState = ParseNormal; - } else if ( c == '\\' ) { - mParserState = ParseInBackslash; - } - break; - case ParseInBackslash: - mParserState = ParseInString; - break; + if (scanUtf(c)) { + QByteArray msg = mBuffer.mid(mParserStartOffset, mParserOffset - mParserStartOffset); + QJsonObject obj = QJsonDocument::fromJson(msg).object(); + if (!obj.isEmpty()) + emit objectReceived(obj); + mBuffer = mBuffer.mid(mParserOffset); + resetParser(); + } + } + break; + case FormatUTF16BE: + for ( ; 2 * mParserOffset < mBuffer.size() ; mParserOffset++ ) { + int16_t c = qFromBigEndian(reinterpret_cast(mBuffer.constData())[mParserOffset]); + if (scanUtf(c)) { + QByteArray msg = mBuffer.mid(mParserStartOffset * 2, 2*(mParserOffset - mParserStartOffset)); + QString s = QTextCodec::codecForName("UTF-16BE")->toUnicode(msg); + QJsonObject obj = QJsonDocument::fromJson(s.toUtf8()).object(); + if (!obj.isEmpty()) + emit objectReceived(obj); + mBuffer = mBuffer.mid(mParserOffset*2); + resetParser(); + } + } + break; + case FormatUTF16LE: + for ( ; 2 * mParserOffset < mBuffer.size() ; mParserOffset++ ) { + int16_t c = qFromLittleEndian(reinterpret_cast(mBuffer.constData())[mParserOffset]); + if (scanUtf(c)) { + QByteArray msg = mBuffer.mid(mParserStartOffset * 2, 2*(mParserOffset - mParserStartOffset)); + QString s = QTextCodec::codecForName("UTF-16LE")->toUnicode(msg); + QJsonObject obj = QJsonDocument::fromJson(s.toUtf8()).object(); + if (!obj.isEmpty()) + emit objectReceived(obj); + mBuffer = mBuffer.mid(mParserOffset*2); + resetParser(); } } break; diff --git a/src/jsonbuffer_p.h b/src/jsonbuffer_p.h index 3d72be9..9bce2ce 100644 --- a/src/jsonbuffer_p.h +++ b/src/jsonbuffer_p.h @@ -67,6 +67,8 @@ signals: private: void processMessages(); + bool scanUtf(int c); + void resetParser(); private: enum UTF8ParsingState { ParseNormal, ParseInString, ParseInBackslash }; diff --git a/src/jsonpipe.cpp b/src/jsonpipe.cpp index 58fb15c..981a25e 100644 --- a/src/jsonpipe.cpp +++ b/src/jsonpipe.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -244,6 +245,12 @@ bool JsonPipe::send(const QJsonObject& object) case FormatUTF8: d->mOutBuffer.append(document.toJson()); break; + case FormatUTF16BE: + d->mOutBuffer.append( QTextCodec::codecForName("UTF-16BE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) ); + break; + case FormatUTF16LE: + d->mOutBuffer.append( QTextCodec::codecForName("UTF-16LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) ); + break; case FormatBSON: { BsonObject bson(document.toVariant().toMap()); diff --git a/src/jsonstream-global.h b/src/jsonstream-global.h index 3edff30..05e6e95 100644 --- a/src/jsonstream-global.h +++ b/src/jsonstream-global.h @@ -67,7 +67,7 @@ QT_BEGIN_NAMESPACE_JSONSTREAM -enum EncodingFormat { FormatUndefined, FormatUTF8, FormatBSON, FormatQBJS }; +enum EncodingFormat { FormatUndefined, FormatUTF8, FormatBSON, FormatQBJS, FormatUTF16BE, FormatUTF16LE }; QT_END_NAMESPACE_JSONSTREAM diff --git a/src/jsonstream.cpp b/src/jsonstream.cpp index 0fe966a..348eb2d 100644 --- a/src/jsonstream.cpp +++ b/src/jsonstream.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include "jsonstream.h" #include "jsonbuffer_p.h" @@ -187,6 +188,12 @@ void JsonStream::send(const QJsonObject& object) case FormatUTF8: sendInternal( document.toJson() ); break; + case FormatUTF16BE: + sendInternal( QTextCodec::codecForName("UTF-16BE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) ); // Chop off BOM + break; + case FormatUTF16LE: + sendInternal( QTextCodec::codecForName("UTF-16LE")->fromUnicode(QString::fromUtf8(document.toJson())).mid(2) ); // Chop off BOM + break; case FormatBSON: { BsonObject bson(document.toVariant().toMap()); diff --git a/tests/auto/jsonstream/testClient/main.cpp b/tests/auto/jsonstream/testClient/main.cpp index a01fa35..79e9057 100644 --- a/tests/auto/jsonstream/testClient/main.cpp +++ b/tests/auto/jsonstream/testClient/main.cpp @@ -83,6 +83,10 @@ Container::Container() mClient->setFormat(FormatBSON); else if (gFormat == "utf" || gFormat == "utf8") mClient->setFormat(FormatUTF8); + else if (gFormat == "utf16be") + mClient->setFormat(FormatUTF16BE); + else if (gFormat == "utf16le") + mClient->setFormat(FormatUTF16LE); if (!mClient->connectLocal(gSocketname)) { qWarning() << "Unable to connect to" << gSocketname; diff --git a/tests/auto/jsonstream/tst_jsonstream.cpp b/tests/auto/jsonstream/tst_jsonstream.cpp index 39b20ff..14f3b08 100644 --- a/tests/auto/jsonstream/tst_jsonstream.cpp +++ b/tests/auto/jsonstream/tst_jsonstream.cpp @@ -367,7 +367,7 @@ void tst_JsonStream::formatTest() { QString socketname = "/tmp/tst_socket"; - QStringList formats = QStringList() << "qbjs" << "bson" << "utf8"; + QStringList formats = QStringList() << "qbjs" << "bson" << "utf8" << "utf16be" << "utf16le"; foreach (const QString& format, formats) { BasicServer server(socketname); @@ -392,6 +392,10 @@ void tst_JsonStream::formatTest() QVERIFY(server.format() == FormatBSON); else if (format == "utf8") QVERIFY(server.format() == FormatUTF8); + else if (format == "utf16be") + QVERIFY(server.format() == FormatUTF16BE); + else if (format == "utf16le") + QVERIFY(server.format() == FormatUTF16LE); else QFAIL("Unrecognized format"); @@ -533,7 +537,7 @@ void tst_JsonStream::pipeTest() void tst_JsonStream::pipeFormatTest() { - QList formats = QList() << FormatUTF8 << FormatBSON << FormatQBJS; + QList formats = QList() << FormatUTF8 << FormatBSON << FormatQBJS << FormatUTF16BE << FormatUTF16LE; foreach (EncodingFormat format, formats) { Pipes pipes; -- cgit v1.2.3