From 1823c8f2ddd0a5c1b4301e7af7109796090a3c9a Mon Sep 17 00:00:00 2001 From: Erik Verbruggen Date: Wed, 16 Dec 2015 14:04:27 +0100 Subject: Fix utf8->utf16 BOM/ZWNBSP decoding. When the byte sequence for a BOM occurs in the middle of a utf8 stream, it is a ZWNBSP. When a ZWNBSP occurs in the middle of a utf8 character sequence, and the SIMD conversion does some work (meaning: the length is at least 16 characters long), it would not recognize the fact some charactes were already decoded. So the conversion would then strip the ZWNBSP out, thinking it's a BOM. The non-SIMD conversion did not have this problem: the very first character conversion would already set the headerdone flag. Change-Id: I39aacf607e2e068107106254021a8042d164f628 Reviewed-by: Thiago Macieira --- tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'tests/auto/corelib/codecs/qtextcodec') diff --git a/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp b/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp index 3aa06d237d..8a9ae0cd72 100644 --- a/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp +++ b/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp @@ -1588,10 +1588,17 @@ void tst_QTextCodec::utf8bom_data() << QString("a"); } - { + { // test the non-SIMD code-path static const ushort data[] = { 0x61, 0xfeff, 0x62 }; - QTest::newRow("middle-bom") - << QByteArray("a\357\273\277b", 5) + QTest::newRow("middle-bom (non SIMD)") + << QByteArray("a\357\273\277b") + << QString::fromUtf16(data, sizeof(data)/sizeof(short)); + } + + { // test the SIMD code-path + static const ushort data[] = { 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0xfeff, 0x6d }; + QTest::newRow("middle-bom (SIMD)") + << QByteArray("abcdefghijkl\357\273\277m") << QString::fromUtf16(data, sizeof(data)/sizeof(short)); } } -- cgit v1.2.3