diff options
Diffstat (limited to 'src/corelib/codecs/qutfcodec.cpp')
| -rw-r--r-- | src/corelib/codecs/qutfcodec.cpp | 670 |
1 files changed, 670 insertions, 0 deletions
diff --git a/src/corelib/codecs/qutfcodec.cpp b/src/corelib/codecs/qutfcodec.cpp new file mode 100644 index 0000000000..c44832936d --- /dev/null +++ b/src/corelib/codecs/qutfcodec.cpp @@ -0,0 +1,670 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qutfcodec_p.h" +#include "qlist.h" +#include "qendian.h" +#include "qchar.h" + +QT_BEGIN_NAMESPACE + +enum { Endian = 0, Data = 1 }; + +static inline bool isUnicodeNonCharacter(uint ucs4) +{ + // Unicode has a couple of "non-characters" that one can use internally, + // but are not allowed to be used for text interchange. + // + // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF, + // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and + // U+FDEF (inclusive) + + return (ucs4 & 0xfffe) == 0xfffe + || (ucs4 - 0xfdd0U) < 16; +} + +QByteArray QUtf8::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state) +{ + uchar replacement = '?'; + int rlen = 3*len; + int surrogate_high = -1; + if (state) { + if (state->flags & QTextCodec::ConvertInvalidToNull) + replacement = 0; + if (!(state->flags & QTextCodec::IgnoreHeader)) + rlen += 3; + if (state->remainingChars) + surrogate_high = state->state_data[0]; + } + + QByteArray rstr; + rstr.resize(rlen); + uchar* cursor = (uchar*)rstr.data(); + const QChar *ch = uc; + int invalid = 0; + if (state && !(state->flags & QTextCodec::IgnoreHeader)) { + *cursor++ = 0xef; + *cursor++ = 0xbb; + *cursor++ = 0xbf; + } + + const QChar *end = ch + len; + while (ch < end) { + uint u = ch->unicode(); + if (surrogate_high >= 0) { + if (u >= 0xdc00 && u < 0xe000) { + u = (surrogate_high - 0xd800)*0x400 + (u - 0xdc00) + 0x10000; + surrogate_high = -1; + } else { + // high surrogate without low + *cursor = replacement; + ++ch; + ++invalid; + surrogate_high = -1; + continue; + } + } else if (u >= 0xdc00 && u < 0xe000) { + // low surrogate without high + *cursor = replacement; + ++ch; + ++invalid; + continue; + } else if (u >= 0xd800 && u < 0xdc00) { + surrogate_high = u; + ++ch; + continue; + } + + if (u < 0x80) { + *cursor++ = (uchar)u; + } else { + if (u < 0x0800) { + *cursor++ = 0xc0 | ((uchar) (u >> 6)); + } else { + // is it one of the Unicode non-characters? + if (isUnicodeNonCharacter(u)) { + *cursor++ = replacement; + ++ch; + ++invalid; + continue; + } + + if (u > 0xffff) { + *cursor++ = 0xf0 | ((uchar) (u >> 18)); + *cursor++ = 0x80 | (((uchar) (u >> 12)) & 0x3f); + } else { + *cursor++ = 0xe0 | (((uchar) (u >> 12)) & 0x3f); + } + *cursor++ = 0x80 | (((uchar) (u >> 6)) & 0x3f); + } + *cursor++ = 0x80 | ((uchar) (u&0x3f)); + } + ++ch; + } + + rstr.resize(cursor - (const uchar*)rstr.constData()); + if (state) { + state->invalidChars += invalid; + state->flags |= QTextCodec::IgnoreHeader; + state->remainingChars = 0; + if (surrogate_high >= 0) { + state->remainingChars = 1; + state->state_data[0] = surrogate_high; + } + } + return rstr; +} + +QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state) +{ + bool headerdone = false; + ushort replacement = QChar::ReplacementCharacter; + int need = 0; + int error = -1; + uint uc = 0; + uint min_uc = 0; + if (state) { + if (state->flags & QTextCodec::IgnoreHeader) + headerdone = true; + if (state->flags & QTextCodec::ConvertInvalidToNull) + replacement = QChar::Null; + need = state->remainingChars; + if (need) { + uc = state->state_data[0]; + min_uc = state->state_data[1]; + } + } + if (!headerdone && len > 3 + && (uchar)chars[0] == 0xef && (uchar)chars[1] == 0xbb && (uchar)chars[2] == 0xbf) { + // starts with a byte order mark + chars += 3; + len -= 3; + headerdone = true; + } + + QString result(need + len + 1, Qt::Uninitialized); // worst case + ushort *qch = (ushort *)result.unicode(); + uchar ch; + int invalid = 0; + + for (int i = 0; i < len; ++i) { + ch = chars[i]; + if (need) { + if ((ch&0xc0) == 0x80) { + uc = (uc << 6) | (ch & 0x3f); + --need; + if (!need) { + // utf-8 bom composes into 0xfeff code point + bool nonCharacter; + if (!headerdone && uc == 0xfeff) { + // don't do anything, just skip the BOM + } else if (!(nonCharacter = isUnicodeNonCharacter(uc)) && uc > 0xffff && uc < 0x110000) { + // surrogate pair + Q_ASSERT((qch - (ushort*)result.unicode()) + 2 < result.length()); + *qch++ = QChar::highSurrogate(uc); + *qch++ = QChar::lowSurrogate(uc); + } else if ((uc < min_uc) || (uc >= 0xd800 && uc <= 0xdfff) || nonCharacter || uc >= 0x110000) { + // error: overlong sequence, UTF16 surrogate or non-character + *qch++ = replacement; + ++invalid; + } else { + *qch++ = uc; + } + headerdone = true; + } + } else { + // error + i = error; + *qch++ = replacement; + ++invalid; + need = 0; + headerdone = true; + } + } else { + if (ch < 128) { + *qch++ = ushort(ch); + headerdone = true; + } else if ((ch & 0xe0) == 0xc0) { + uc = ch & 0x1f; + need = 1; + error = i; + min_uc = 0x80; + headerdone = true; + } else if ((ch & 0xf0) == 0xe0) { + uc = ch & 0x0f; + need = 2; + error = i; + min_uc = 0x800; + } else if ((ch&0xf8) == 0xf0) { + uc = ch & 0x07; + need = 3; + error = i; + min_uc = 0x10000; + headerdone = true; + } else { + // error + *qch++ = replacement; + ++invalid; + headerdone = true; + } + } + } + if (!state && need > 0) { + // unterminated UTF sequence + for (int i = error; i < len; ++i) { + *qch++ = replacement; + ++invalid; + } + } + result.truncate(qch - (ushort *)result.unicode()); + if (state) { + state->invalidChars += invalid; + state->remainingChars = need; + if (headerdone) + state->flags |= QTextCodec::IgnoreHeader; + state->state_data[0] = need ? uc : 0; + state->state_data[1] = need ? min_uc : 0; + } + return result; +} + +QByteArray QUtf16::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state, DataEndianness e) +{ + DataEndianness endian = e; + int length = 2*len; + if (!state || (!(state->flags & QTextCodec::IgnoreHeader))) { + length += 2; + } + if (e == DetectEndianness) { + endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; + } + + QByteArray d; + d.resize(length); + char *data = d.data(); + if (!state || !(state->flags & QTextCodec::IgnoreHeader)) { + QChar bom(QChar::ByteOrderMark); + if (endian == BigEndianness) { + data[0] = bom.row(); + data[1] = bom.cell(); + } else { + data[0] = bom.cell(); + data[1] = bom.row(); + } + data += 2; + } + if (endian == BigEndianness) { + for (int i = 0; i < len; ++i) { + *(data++) = uc[i].row(); + *(data++) = uc[i].cell(); + } + } else { + for (int i = 0; i < len; ++i) { + *(data++) = uc[i].cell(); + *(data++) = uc[i].row(); + } + } + + if (state) { + state->remainingChars = 0; + state->flags |= QTextCodec::IgnoreHeader; + } + return d; +} + +QString QUtf16::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state, DataEndianness e) +{ + DataEndianness endian = e; + bool half = false; + uchar buf = 0; + bool headerdone = false; + if (state) { + headerdone = state->flags & QTextCodec::IgnoreHeader; + if (endian == DetectEndianness) + endian = (DataEndianness)state->state_data[Endian]; + if (state->remainingChars) { + half = true; + buf = state->state_data[Data]; + } + } + if (headerdone && endian == DetectEndianness) + endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; + + QString result(len, Qt::Uninitialized); // worst case + QChar *qch = (QChar *)result.unicode(); + while (len--) { + if (half) { + QChar ch; + if (endian == LittleEndianness) { + ch.setRow(*chars++); + ch.setCell(buf); + } else { + ch.setRow(buf); + ch.setCell(*chars++); + } + if (!headerdone) { + headerdone = true; + if (endian == DetectEndianness) { + if (ch == QChar::ByteOrderSwapped) { + endian = LittleEndianness; + } else if (ch == QChar::ByteOrderMark) { + endian = BigEndianness; + } else { + if (QSysInfo::ByteOrder == QSysInfo::BigEndian) { + endian = BigEndianness; + } else { + endian = LittleEndianness; + ch = QChar((ch.unicode() >> 8) | ((ch.unicode() & 0xff) << 8)); + } + *qch++ = ch; + } + } else if (ch != QChar::ByteOrderMark) { + *qch++ = ch; + } + } else { + *qch++ = ch; + } + half = false; + } else { + buf = *chars++; + half = true; + } + } + result.truncate(qch - result.unicode()); + + if (state) { + if (headerdone) + state->flags |= QTextCodec::IgnoreHeader; + state->state_data[Endian] = endian; + if (half) { + state->remainingChars = 1; + state->state_data[Data] = buf; + } else { + state->remainingChars = 0; + state->state_data[Data] = 0; + } + } + return result; +} + +QByteArray QUtf32::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state, DataEndianness e) +{ + DataEndianness endian = e; + int length = 4*len; + if (!state || (!(state->flags & QTextCodec::IgnoreHeader))) { + length += 4; + } + if (e == DetectEndianness) { + endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; + } + + QByteArray d(length, Qt::Uninitialized); + char *data = d.data(); + if (!state || !(state->flags & QTextCodec::IgnoreHeader)) { + if (endian == BigEndianness) { + data[0] = 0; + data[1] = 0; + data[2] = (char)0xfe; + data[3] = (char)0xff; + } else { + data[0] = (char)0xff; + data[1] = (char)0xfe; + data[2] = 0; + data[3] = 0; + } + data += 4; + } + if (endian == BigEndianness) { + for (int i = 0; i < len; ++i) { + uint cp = uc[i].unicode(); + if (uc[i].isHighSurrogate() && i < len - 1) + cp = QChar::surrogateToUcs4(cp, uc[++i].unicode()); + *(data++) = cp >> 24; + *(data++) = (cp >> 16) & 0xff; + *(data++) = (cp >> 8) & 0xff; + *(data++) = cp & 0xff; + } + } else { + for (int i = 0; i < len; ++i) { + uint cp = uc[i].unicode(); + if (uc[i].isHighSurrogate() && i < len - 1) + cp = QChar::surrogateToUcs4(cp, uc[++i].unicode()); + *(data++) = cp & 0xff; + *(data++) = (cp >> 8) & 0xff; + *(data++) = (cp >> 16) & 0xff; + *(data++) = cp >> 24; + } + } + + if (state) { + state->remainingChars = 0; + state->flags |= QTextCodec::IgnoreHeader; + } + return d; +} + +QString QUtf32::convertToUnicode(const char *chars, int len, QTextCodec::ConverterState *state, DataEndianness e) +{ + DataEndianness endian = e; + uchar tuple[4]; + int num = 0; + bool headerdone = false; + if (state) { + headerdone = state->flags & QTextCodec::IgnoreHeader; + if (endian == DetectEndianness) { + endian = (DataEndianness)state->state_data[Endian]; + } + num = state->remainingChars; + memcpy(tuple, &state->state_data[Data], 4); + } + if (headerdone && endian == DetectEndianness) + endian = (QSysInfo::ByteOrder == QSysInfo::BigEndian) ? BigEndianness : LittleEndianness; + + QString result; + result.resize((num + len) >> 2 << 1); // worst case + QChar *qch = (QChar *)result.unicode(); + + const char *end = chars + len; + while (chars < end) { + tuple[num++] = *chars++; + if (num == 4) { + if (!headerdone) { + if (endian == DetectEndianness) { + if (endian == DetectEndianness) { + if (tuple[0] == 0xff && tuple[1] == 0xfe && tuple[2] == 0 && tuple[3] == 0 && endian != BigEndianness) { + endian = LittleEndianness; + num = 0; + continue; + } else if (tuple[0] == 0 && tuple[1] == 0 && tuple[2] == 0xfe && tuple[3] == 0xff && endian != LittleEndianness) { + endian = BigEndianness; + num = 0; + continue; + } else if (QSysInfo::ByteOrder == QSysInfo::BigEndian) { + endian = BigEndianness; + } else { + endian = LittleEndianness; + } + } + } else if (((endian == BigEndianness) ? qFromBigEndian<quint32>(tuple) : qFromLittleEndian<quint32>(tuple)) == QChar::ByteOrderMark) { + num = 0; + continue; + } + } + uint code = (endian == BigEndianness) ? qFromBigEndian<quint32>(tuple) : qFromLittleEndian<quint32>(tuple); + if (code >= 0x10000) { + *qch++ = QChar::highSurrogate(code); + *qch++ = QChar::lowSurrogate(code); + } else { + *qch++ = code; + } + num = 0; + } + } + result.truncate(qch - result.unicode()); + + if (state) { + if (headerdone) + state->flags |= QTextCodec::IgnoreHeader; + state->state_data[Endian] = endian; + state->remainingChars = num; + memcpy(&state->state_data[Data], tuple, 4); + } + return result; +} + + +#ifndef QT_NO_TEXTCODEC + +QUtf8Codec::~QUtf8Codec() +{ +} + +QByteArray QUtf8Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const +{ + return QUtf8::convertFromUnicode(uc, len, state); +} + +void QUtf8Codec::convertToUnicode(QString *target, const char *chars, int len, ConverterState *state) const +{ + *target += QUtf8::convertToUnicode(chars, len, state); +} + +QString QUtf8Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const +{ + return QUtf8::convertToUnicode(chars, len, state); +} + +QByteArray QUtf8Codec::name() const +{ + return "UTF-8"; +} + +int QUtf8Codec::mibEnum() const +{ + return 106; +} + +QUtf16Codec::~QUtf16Codec() +{ +} + +QByteArray QUtf16Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const +{ + return QUtf16::convertFromUnicode(uc, len, state, e); +} + +QString QUtf16Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const +{ + return QUtf16::convertToUnicode(chars, len, state, e); +} + +int QUtf16Codec::mibEnum() const +{ + return 1015; +} + +QByteArray QUtf16Codec::name() const +{ + return "UTF-16"; +} + +QList<QByteArray> QUtf16Codec::aliases() const +{ + return QList<QByteArray>(); +} + +int QUtf16BECodec::mibEnum() const +{ + return 1013; +} + +QByteArray QUtf16BECodec::name() const +{ + return "UTF-16BE"; +} + +QList<QByteArray> QUtf16BECodec::aliases() const +{ + QList<QByteArray> list; + return list; +} + +int QUtf16LECodec::mibEnum() const +{ + return 1014; +} + +QByteArray QUtf16LECodec::name() const +{ + return "UTF-16LE"; +} + +QList<QByteArray> QUtf16LECodec::aliases() const +{ + QList<QByteArray> list; + return list; +} + +QUtf32Codec::~QUtf32Codec() +{ +} + +QByteArray QUtf32Codec::convertFromUnicode(const QChar *uc, int len, ConverterState *state) const +{ + return QUtf32::convertFromUnicode(uc, len, state, e); +} + +QString QUtf32Codec::convertToUnicode(const char *chars, int len, ConverterState *state) const +{ + return QUtf32::convertToUnicode(chars, len, state, e); +} + +int QUtf32Codec::mibEnum() const +{ + return 1017; +} + +QByteArray QUtf32Codec::name() const +{ + return "UTF-32"; +} + +QList<QByteArray> QUtf32Codec::aliases() const +{ + QList<QByteArray> list; + return list; +} + +int QUtf32BECodec::mibEnum() const +{ + return 1018; +} + +QByteArray QUtf32BECodec::name() const +{ + return "UTF-32BE"; +} + +QList<QByteArray> QUtf32BECodec::aliases() const +{ + QList<QByteArray> list; + return list; +} + +int QUtf32LECodec::mibEnum() const +{ + return 1019; +} + +QByteArray QUtf32LECodec::name() const +{ + return "UTF-32LE"; +} + +QList<QByteArray> QUtf32LECodec::aliases() const +{ + QList<QByteArray> list; + return list; +} + +#endif //QT_NO_TEXTCODEC + +QT_END_NAMESPACE |