From add2bf739ae96603cb919b908cbb53c00d0628cc Mon Sep 17 00:00:00 2001 From: Kurt Pattyn Date: Sun, 6 Oct 2013 11:40:47 +0200 Subject: Allow non-character codes in utf8 strings Changed the processing of non-character code handling in the UTF8 codec. Non-character codes are now accepted in QStrings, QUrls and QJson strings. Unit tests were adapted accordingly. For more info about non-character codes, see: http://www.unicode.org/versions/corrigendum9.html [ChangeLog][QtCore][QUtf8] UTF-8 now accepts non-character unicode points; these are not replaced by the replacement character anymore [ChangeLog][QtCore][QUrl] QUrl now fully accepts non-character unicode points; they are encoded as percent characters; they can also be pretty decoded [ChangeLog][QtCore][QJson] The Writer and the Parser now fully accept non-character unicode points. Change-Id: I77cf4f0e6210741eac8082912a0b6118eced4f77 Task-number: QTBUG-33229 Reviewed-by: Lars Knoll Reviewed-by: Thiago Macieira --- .../corelib/codecs/qtextcodec/tst_qtextcodec.cpp | 54 +++++++++++----------- 1 file changed, 28 insertions(+), 26 deletions(-) (limited to 'tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp') diff --git a/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp b/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp index dd557b8d21..8e1b3cf3b2 100644 --- a/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp +++ b/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp @@ -66,9 +66,9 @@ private slots: void codecForLocale(); void asciiToIscii() const; - void flagCodepointFFFF() const; + void nonFlaggedCodepointFFFF() const; void flagF7808080() const; - void flagEFBFBF() const; + void nonFlaggedEFBFBF() const; void decode0D() const; void aliasForUTF16() const; void mibForTSCII() const; @@ -409,9 +409,9 @@ void tst_QTextCodec::asciiToIscii() const } } -void tst_QTextCodec::flagCodepointFFFF() const +void tst_QTextCodec::nonFlaggedCodepointFFFF() const { - // This is an invalid Unicode codepoint. + //Check that the code point 0xFFFF (=non-character code 0xEFBFBF) is not flagged const QChar ch(0xFFFF); QString input(ch); @@ -419,12 +419,11 @@ void tst_QTextCodec::flagCodepointFFFF() const QVERIFY(codec); const QByteArray asDecoded(codec->fromUnicode(input)); - QCOMPARE(asDecoded, QByteArray("?")); + QCOMPARE(asDecoded, QByteArray("\357\277\277")); QByteArray ffff("\357\277\277"); QTextCodec::ConverterState state(QTextCodec::ConvertInvalidToNull); - QVERIFY(codec->toUnicode(ffff.constData(), ffff.length(), &state) == QChar(0)); - QVERIFY(codec->toUnicode(ffff) == QChar(0xfffd)); + QVERIFY(codec->toUnicode(ffff.constData(), ffff.length(), &state) == QByteArray::fromHex("EFBFBF")); } void tst_QTextCodec::flagF7808080() const @@ -460,13 +459,16 @@ void tst_QTextCodec::flagF7808080() const QVERIFY(codec->toUnicode(input.constData(), input.length(), &state) == QChar(0)); } -void tst_QTextCodec::flagEFBFBF() const +void tst_QTextCodec::nonFlaggedEFBFBF() const { - QByteArray invalidInput; - invalidInput.resize(3); - invalidInput[0] = char(0xEF); - invalidInput[1] = char(0xBF); - invalidInput[2] = char(0xBF); + /* Check that the codec does NOT flag EFBFBF. + * This is a regression test; see QTBUG-33229 + */ + QByteArray validInput; + validInput.resize(3); + validInput[0] = char(0xEF); + validInput[1] = char(0xBF); + validInput[2] = char(0xBF); const QTextCodec *const codec = QTextCodec::codecForMib(106); // UTF-8 QVERIFY(codec); @@ -474,21 +476,20 @@ void tst_QTextCodec::flagEFBFBF() const { //QVERIFY(!codec->canEncode(QChar(0xFFFF))); QTextCodec::ConverterState state(QTextCodec::ConvertInvalidToNull); - QVERIFY(codec->toUnicode(invalidInput.constData(), invalidInput.length(), &state) == QChar(0)); + QVERIFY(codec->toUnicode(validInput.constData(), validInput.length(), &state) == QByteArray::fromHex("EFBFBF")); QByteArray start(""); } - /* When 0xEFBFBF is preceded by what seems to be an arbitrary character, - * QTextCodec fails to flag it. */ + // Check that 0xEFBFBF is correctly decoded when preceded by an arbitrary character { QByteArray start("B"); - start.append(invalidInput); + start.append(validInput); QTextCodec::ConverterState state(QTextCodec::ConvertInvalidToNull); - QVERIFY(codec->toUnicode(start.constData(), start.length(), &state) == QString::fromLatin1("B\0", 2)); + QVERIFY(codec->toUnicode(start.constData(), start.length(), &state) == QByteArray("B").append(QByteArray::fromHex("EFBFBF"))); } } @@ -674,13 +675,12 @@ void tst_QTextCodec::utf8Codec_data() str = QChar(0x7ff); QTest::newRow("http://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html 2.2.2") << utf8 << str << -1; - // 2.2.3 U+000FFFF + // 2.2.3 U+000FFFF - non-character code utf8.clear(); utf8 += char(0xef); utf8 += char(0xbf); utf8 += char(0xbf); - str.clear(); - str += QChar::ReplacementCharacter; + str = QString::fromUtf8(utf8); QTest::newRow("http://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html 2.2.3") << utf8 << str << -1; // 2.2.4 U+001FFFFF @@ -1535,20 +1535,22 @@ void tst_QTextCodec::utf8Codec_data() str += QChar(QChar::ReplacementCharacter); QTest::newRow("http://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html 5.2.8") << utf8 << str << -1; - // 5.3.1 + // 5.3.1 - non-character code utf8.clear(); utf8 += char(0xef); utf8 += char(0xbf); utf8 += char(0xbe); - str = QChar(QChar::ReplacementCharacter); + //str = QChar(QChar::ReplacementCharacter); + str = QString::fromUtf8(utf8); QTest::newRow("http://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html 5.3.1") << utf8 << str << -1; - // 5.3.2 + // 5.3.2 - non-character code utf8.clear(); utf8 += char(0xef); utf8 += char(0xbf); utf8 += char(0xbf); - str = QChar(QChar::ReplacementCharacter); + //str = QChar(QChar::ReplacementCharacter); + str = QString::fromUtf8(utf8); QTest::newRow("http://www.w3.org/2001/06/utf-8-wrong/UTF-8-test.html 5.3.2") << utf8 << str << -1; } -- cgit v1.2.3