diff options
Diffstat (limited to 'tests/auto/corelib/io/qurlinternal/utf8data.cpp')
| -rw-r--r-- | tests/auto/corelib/io/qurlinternal/utf8data.cpp | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/tests/auto/corelib/io/qurlinternal/utf8data.cpp b/tests/auto/corelib/io/qurlinternal/utf8data.cpp new file mode 100644 index 0000000000..ef02761a0e --- /dev/null +++ b/tests/auto/corelib/io/qurlinternal/utf8data.cpp @@ -0,0 +1,160 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Copyright (C) 2018 Intel Corporation. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include <QtTest/QtTest> + +void loadInvalidUtf8Rows() +{ + // Wrong continuations + QTest::newRow("bad-continuation-1char") << QByteArray("\x80"); + QTest::newRow("bad-continuation-2chars-1") << QByteArray("\xC2\xC0"); + QTest::newRow("bad-continuation-2chars-2") << QByteArray("\xC3\xDF"); + QTest::newRow("bad-continuation-2chars-3") << QByteArray("\xC7\xF0"); + QTest::newRow("bad-continuation-3chars-1") << QByteArray("\xE0\xA0\xC0"); + QTest::newRow("bad-continuation-3chars-2") << QByteArray("\xE0\xC0\xA0"); + QTest::newRow("bad-continuation-4chars-1") << QByteArray("\xF0\x90\x80\xC0"); + QTest::newRow("bad-continuation-4chars-2") << QByteArray("\xF0\x90\xC0\x80"); + QTest::newRow("bad-continuation-4chars-3") << QByteArray("\xF0\xC0\x80\x80"); + + // Too short + QTest::newRow("too-short-2chars") << QByteArray("\xC2"); + QTest::newRow("too-short-3chars-1") << QByteArray("\xE0"); + QTest::newRow("too-short-3chars-2") << QByteArray("\xE0\xA0"); + QTest::newRow("too-short-4chars-1") << QByteArray("\xF0"); + QTest::newRow("too-short-4chars-2") << QByteArray("\xF0\x90"); + QTest::newRow("too-short-4chars-3") << QByteArray("\xF0\x90\x80"); + + // Surrogate pairs must now be present either + // U+D800: 1101 10 0000 00 0000 + // encoding: xxxz:1101 xz10:0000 xz00:0000 + QTest::newRow("hi-surrogate") << QByteArray("\xED\xA0\x80"); + // U+DC00: 1101 11 0000 00 0000 + // encoding: xxxz:1101 xz11:0000 xz00:0000 + QTest::newRow("lo-surrogate") << QByteArray("\xED\xB0\x80"); + + // not even in pair: + QTest::newRow("surrogate-pair") << QByteArray("\xED\xA0\x80\xED\xB0\x80"); + + // Characters outside the Unicode range: + // 0x110000: 00 0100 01 0000 00 0000 00 0000 + // encoding: xxxx:z100 xz01:0000 xz00:0000 xz00:0000 + QTest::newRow("non-unicode-1") << QByteArray("\xF4\x90\x80\x80"); + // 0x200000: 00 1000 00 0000 00 0000 00 0000 + // encoding: xxxx:xz00 xz00:1000 xz00:0000 xz00:0000 xz00:0000 + QTest::newRow("non-unicode-2") << QByteArray("\xF8\x88\x80\x80\x80"); + // 0x04000000: 0100 00 0000 00 0000 00 0000 00 0000 + // encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001 + QTest::newRow("non-unicode-3") << QByteArray("\xFC\x84\x80\x80\x80\x80"); + // 0x7fffffff: 1 11 1111 11 1111 11 1111 11 1111 11 1111 + // encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001 + QTest::newRow("non-unicode-4") << QByteArray("\xFD\xBF\xBF\xBF\xBF\xBF"); + + // As seen above, 0xFE and 0xFF never appear: + QTest::newRow("fe") << QByteArray("\xFE"); + QTest::newRow("fe-bis") << QByteArray("\xFE\xBF\xBF\xBF\xBF\xBF\xBF"); + QTest::newRow("ff") << QByteArray("\xFF"); + QTest::newRow("ff-bis") << QByteArray("\xFF\xBF\xBF\xBF\xBF\xBF\xBF\xBF"); + + // some combinations in UTF-8 are invalid even though they have the proper bits set + // these are known as overlong sequences + + // "A": U+0041: 01 00 0001 + // overlong 2: xxz0:0001 xz00:0001 + QTest::newRow("overlong-1-2") << QByteArray("\xC1\x81"); + // overlong 3: xxxz:0000 xz00:0001 xz00:0001 + QTest::newRow("overlong-1-3") << QByteArray("\xE0\x81\x81"); + // overlong 4: xxxx:z000 xz00:0000 xz00:0001 xz00:0001 + QTest::newRow("overlong-1-4") << QByteArray("\xF0\x80\x81\x81"); + // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0001 xz00:0001 + QTest::newRow("overlong-1-5") << QByteArray("\xF8\x80\x80\x81\x81"); + // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0001 xz00:0001 + QTest::newRow("overlong-1-6") << QByteArray("\xFC\x80\x80\x80\x81\x81"); + + // U+0080: 10 00 0000 + // proper encoding: xxz0:0010 xz00:0000 + // overlong 3: xxxz:0000 xz00:0010 xz00:0000 + QTest::newRow("overlong-2-3") << QByteArray("\xE0\x82\x80"); + // overlong 4: xxxx:z000 xz00:0000 xz00:0010 xz00:0000 + QTest::newRow("overlong-2-4") << QByteArray("\xF0\x80\x82\x80"); + // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0010 xz00:0000 + QTest::newRow("overlong-2-5") << QByteArray("\xF8\x80\x80\x82\x80"); + // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0010 xz00:0000 + QTest::newRow("overlong-2-6") << QByteArray("\xFC\x80\x80\x80\x82\x80"); + + // U+0800: 10 0000 00 0000 + // proper encoding: xxxz:0000 xz10:0000 xz00:0000 + // overlong 4: xxxx:z000 xz00:0000 xz10:0000 xz00:0000 + QTest::newRow("overlong-3-4") << QByteArray("\xF0\x80\xA0\x80"); + // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz10:0000 xz00:0000 + QTest::newRow("overlong-3-5") << QByteArray("\xF8\x80\x80\xA0\x80"); + // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz10:0000 xz00:0000 + QTest::newRow("overlong-3-6") << QByteArray("\xFC\x80\x80\x80\xA0\x80"); + + // U+010000: 00 0100 00 0000 00 0000 + // proper encoding: xxxx:z000 xz00:0100 xz00:0000 xz00:0000 + // overlong 5: xxxx:xz00 xz00:0000 xz00:0100 xz00:0000 xz00:0000 + QTest::newRow("overlong-4-5") << QByteArray("\xF8\x80\x84\x80\x80"); + // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0100 xz00:0000 xz00:0000 + QTest::newRow("overlong-4-6") << QByteArray("\xFC\x80\x80\x84\x80\x80"); + +} + +void loadNonCharactersRows() +{ + // Unicode has a couple of "non-characters" that one can use internally + // These characters are allowed for text-interchange (see http://www.unicode.org/versions/corrigendum9.html) + // + // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF, + // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and + // U+FDEF (inclusive) + + // U+FDD0 through U+FDEF + for (int i = 0; i < 16; ++i) { + char utf8[] = { char(0357), char(0267), char(0220 + i), 0 }; + QString utf16 = QChar(0xfdd0 + i); + QTest::newRow(qPrintable(QString::number(0xfdd0 + i, 16))) << QByteArray(utf8) << utf16; + } + + // the last two in Planes 1 through 16 + for (uint plane = 1; plane <= 16; ++plane) { + for (uint lower = 0xfffe; lower < 0x10000; ++lower) { + uint ucs4 = (plane << 16) | lower; + char utf8[] = { char(0xf0 | uchar(ucs4 >> 18)), + char(0x80 | (uchar(ucs4 >> 12) & 0x3f)), + char(0x80 | (uchar(ucs4 >> 6) & 0x3f)), + char(0x80 | (uchar(ucs4) & 0x3f)), + 0 }; + const auto utf16 = QChar::fromUcs4(ucs4); + + QTest::newRow(qPrintable(QString::number(ucs4, 16))) << QByteArray(utf8) << QStringView{utf16}.toString(); + } + } + + QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE") << QString(QChar(0xfffe)); + QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF") << QString(QChar(0xffff)); +} |