1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
// Copyright (C) 2018 The Qt Company Ltd.
// Copyright (C) 2018 Intel Corporation.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include <QTest>
void loadInvalidUtf8Rows()
{
// Wrong continuations
QTest::newRow("bad-continuation-1char") << QByteArray("\x80");
QTest::newRow("bad-continuation-2chars-1") << QByteArray("\xC2\xC0");
QTest::newRow("bad-continuation-2chars-2") << QByteArray("\xC3\xDF");
QTest::newRow("bad-continuation-2chars-3") << QByteArray("\xC7\xF0");
QTest::newRow("bad-continuation-3chars-1") << QByteArray("\xE0\xA0\xC0");
QTest::newRow("bad-continuation-3chars-2") << QByteArray("\xE0\xC0\xA0");
QTest::newRow("bad-continuation-4chars-1") << QByteArray("\xF0\x90\x80\xC0");
QTest::newRow("bad-continuation-4chars-2") << QByteArray("\xF0\x90\xC0\x80");
QTest::newRow("bad-continuation-4chars-3") << QByteArray("\xF0\xC0\x80\x80");
// Too short
QTest::newRow("too-short-2chars") << QByteArray("\xC2");
QTest::newRow("too-short-3chars-1") << QByteArray("\xE0");
QTest::newRow("too-short-3chars-2") << QByteArray("\xE0\xA0");
QTest::newRow("too-short-4chars-1") << QByteArray("\xF0");
QTest::newRow("too-short-4chars-2") << QByteArray("\xF0\x90");
QTest::newRow("too-short-4chars-3") << QByteArray("\xF0\x90\x80");
// Surrogate pairs must now be present either
// U+D800: 1101 10 0000 00 0000
// encoding: xxxz:1101 xz10:0000 xz00:0000
QTest::newRow("hi-surrogate") << QByteArray("\xED\xA0\x80");
// U+DC00: 1101 11 0000 00 0000
// encoding: xxxz:1101 xz11:0000 xz00:0000
QTest::newRow("lo-surrogate") << QByteArray("\xED\xB0\x80");
// not even in pair:
QTest::newRow("surrogate-pair") << QByteArray("\xED\xA0\x80\xED\xB0\x80");
// Characters outside the Unicode range:
// 0x110000: 00 0100 01 0000 00 0000 00 0000
// encoding: xxxx:z100 xz01:0000 xz00:0000 xz00:0000
QTest::newRow("non-unicode-1") << QByteArray("\xF4\x90\x80\x80");
// 0x200000: 00 1000 00 0000 00 0000 00 0000
// encoding: xxxx:xz00 xz00:1000 xz00:0000 xz00:0000 xz00:0000
QTest::newRow("non-unicode-2") << QByteArray("\xF8\x88\x80\x80\x80");
// 0x04000000: 0100 00 0000 00 0000 00 0000 00 0000
// encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001
QTest::newRow("non-unicode-3") << QByteArray("\xFC\x84\x80\x80\x80\x80");
// 0x7fffffff: 1 11 1111 11 1111 11 1111 11 1111 11 1111
// encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001
QTest::newRow("non-unicode-4") << QByteArray("\xFD\xBF\xBF\xBF\xBF\xBF");
// As seen above, 0xFE and 0xFF never appear:
QTest::newRow("fe") << QByteArray("\xFE");
QTest::newRow("fe-bis") << QByteArray("\xFE\xBF\xBF\xBF\xBF\xBF\xBF");
QTest::newRow("ff") << QByteArray("\xFF");
QTest::newRow("ff-bis") << QByteArray("\xFF\xBF\xBF\xBF\xBF\xBF\xBF\xBF");
// some combinations in UTF-8 are invalid even though they have the proper bits set
// these are known as overlong sequences
// "A": U+0041: 01 00 0001
// overlong 2: xxz0:0001 xz00:0001
QTest::newRow("overlong-1-2") << QByteArray("\xC1\x81");
// overlong 3: xxxz:0000 xz00:0001 xz00:0001
QTest::newRow("overlong-1-3") << QByteArray("\xE0\x81\x81");
// overlong 4: xxxx:z000 xz00:0000 xz00:0001 xz00:0001
QTest::newRow("overlong-1-4") << QByteArray("\xF0\x80\x81\x81");
// overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0001 xz00:0001
QTest::newRow("overlong-1-5") << QByteArray("\xF8\x80\x80\x81\x81");
// overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0001 xz00:0001
QTest::newRow("overlong-1-6") << QByteArray("\xFC\x80\x80\x80\x81\x81");
// U+0080: 10 00 0000
// proper encoding: xxz0:0010 xz00:0000
// overlong 3: xxxz:0000 xz00:0010 xz00:0000
QTest::newRow("overlong-2-3") << QByteArray("\xE0\x82\x80");
// overlong 4: xxxx:z000 xz00:0000 xz00:0010 xz00:0000
QTest::newRow("overlong-2-4") << QByteArray("\xF0\x80\x82\x80");
// overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0010 xz00:0000
QTest::newRow("overlong-2-5") << QByteArray("\xF8\x80\x80\x82\x80");
// overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0010 xz00:0000
QTest::newRow("overlong-2-6") << QByteArray("\xFC\x80\x80\x80\x82\x80");
// U+0800: 10 0000 00 0000
// proper encoding: xxxz:0000 xz10:0000 xz00:0000
// overlong 4: xxxx:z000 xz00:0000 xz10:0000 xz00:0000
QTest::newRow("overlong-3-4") << QByteArray("\xF0\x80\xA0\x80");
// overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz10:0000 xz00:0000
QTest::newRow("overlong-3-5") << QByteArray("\xF8\x80\x80\xA0\x80");
// overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz10:0000 xz00:0000
QTest::newRow("overlong-3-6") << QByteArray("\xFC\x80\x80\x80\xA0\x80");
// U+010000: 00 0100 00 0000 00 0000
// proper encoding: xxxx:z000 xz00:0100 xz00:0000 xz00:0000
// overlong 5: xxxx:xz00 xz00:0000 xz00:0100 xz00:0000 xz00:0000
QTest::newRow("overlong-4-5") << QByteArray("\xF8\x80\x84\x80\x80");
// overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0100 xz00:0000 xz00:0000
QTest::newRow("overlong-4-6") << QByteArray("\xFC\x80\x80\x84\x80\x80");
}
void loadNonCharactersRows()
{
// Unicode has a couple of "non-characters" that one can use internally
// These characters are allowed for text-interchange (see http://www.unicode.org/versions/corrigendum9.html)
//
// Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF,
// U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and
// U+FDEF (inclusive)
// U+FDD0 through U+FDEF
for (int i = 0; i < 16; ++i) {
char utf8[] = { char(0357), char(0267), char(0220 + i), 0 };
QString utf16 = QChar(0xfdd0 + i);
QTest::newRow(qPrintable(QString::number(0xfdd0 + i, 16))) << QByteArray(utf8) << utf16;
}
// the last two in Planes 1 through 16
for (uint plane = 1; plane <= 16; ++plane) {
for (uint lower = 0xfffe; lower < 0x10000; ++lower) {
uint ucs4 = (plane << 16) | lower;
char utf8[] = { char(0xf0 | uchar(ucs4 >> 18)),
char(0x80 | (uchar(ucs4 >> 12) & 0x3f)),
char(0x80 | (uchar(ucs4 >> 6) & 0x3f)),
char(0x80 | (uchar(ucs4) & 0x3f)),
0 };
const auto utf16 = QChar::fromUcs4(ucs4);
QTest::newRow(qPrintable(QString::number(ucs4, 16))) << QByteArray(utf8) << QStringView{utf16}.toString();
}
}
QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE") << QString(QChar(0xfffe));
QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF") << QString(QChar(0xffff));
}
|
