summaryrefslogtreecommitdiffstats
path: root/src/corelib/text/qstringconverter.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/corelib/text/qstringconverter.h')
-rw-r--r--src/corelib/text/qstringconverter.h374
1 files changed, 128 insertions, 246 deletions
diff --git a/src/corelib/text/qstringconverter.h b/src/corelib/text/qstringconverter.h
index 8766968e6d..055019836a 100644
--- a/src/corelib/text/qstringconverter.h
+++ b/src/corelib/text/qstringconverter.h
@@ -1,247 +1,75 @@
-/****************************************************************************
-**
-** Copyright (C) 2020 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the QtCore module of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:LGPL$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU Lesser General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU Lesser
-** General Public License version 3 as published by the Free Software
-** Foundation and appearing in the file LICENSE.LGPL3 included in the
-** packaging of this file. Please review the following information to
-** ensure the GNU Lesser General Public License version 3 requirements
-** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 2.0 or (at your option) the GNU General
-** Public license version 3 or any later version approved by the KDE Free
-** Qt Foundation. The licenses are as published by the Free Software
-** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-2.0.html and
-** https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
+// Copyright (C) 2020 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#if 0
+// keep existing syncqt header working after the move of the class
+// into qstringconverter_base
+#pragma qt_class(QStringConverter)
+#pragma qt_class(QStringConverterBase)
+#endif
#ifndef QSTRINGCONVERTER_H
#define QSTRINGCONVERTER_H
+#include <QtCore/qstringconverter_base.h>
#include <QtCore/qstring.h>
-#if defined(QT_USE_FAST_OPERATOR_PLUS) || defined(QT_USE_QSTRINGBUILDER)
#include <QtCore/qstringbuilder.h>
-#endif
-
-#include <optional>
QT_BEGIN_NAMESPACE
-// work around a compiler bug in GCC 7
-#if defined(Q_CC_GNU) && __GNUC__ == 7
-#define QSTRINGCONVERTER_CONSTEXPR
-#else
-#define QSTRINGCONVERTER_CONSTEXPR constexpr
-#endif
-
-class QStringConverterBase
-{
-public:
- enum class Flag {
- Default = 0,
- Stateless = 0x1,
- ConvertInvalidToNull = 0x2,
- WriteBom = 0x4,
- ConvertInitialBom = 0x8
- };
- Q_DECLARE_FLAGS(Flags, Flag)
-
- struct State {
- constexpr State(Flags f = Flag::Default)
- : flags(f), state_data{0, 0, 0, 0} {}
- ~State() { clear(); }
- State(State &&other)
- : flags(other.flags),
- remainingChars(other.remainingChars),
- invalidChars(other.invalidChars),
- d{other.d[0], other.d[1]},
- clearFn(other.clearFn)
- { other.clearFn = nullptr; }
- State &operator=(State &&other)
- {
- clear();
- flags = other.flags;
- remainingChars = other.remainingChars;
- invalidChars = other.invalidChars;
- d[0] = other.d[0];
- d[1] = other.d[1];
- clearFn = other.clearFn;
- other.clearFn = nullptr;
- return *this;
- }
- Q_CORE_EXPORT void clear();
-
- Flags flags;
- int internalState = 0;
- qsizetype remainingChars = 0;
- qsizetype invalidChars = 0;
-
- union {
- uint state_data[4];
- void *d[2];
- };
- using ClearDataFn = void (*)(State *);
- ClearDataFn clearFn = nullptr;
- private:
- Q_DISABLE_COPY(State)
- };
-};
-Q_DECLARE_OPERATORS_FOR_FLAGS(QStringConverterBase::Flags)
-
-class QStringConverter : public QStringConverterBase
-{
-public:
-
- enum Encoding {
- Utf8,
- Utf16,
- Utf16LE,
- Utf16BE,
- Utf32,
- Utf32LE,
- Utf32BE,
- Latin1,
- System,
- LastEncoding = System
- };
-#ifdef Q_QDOC
- // document the flags here
- enum class Flag {
- Default = 0,
- Stateless = 0x1,
- ConvertInvalidToNull = 0x2,
- WriteBom = 0x4,
- ConvertInitialBom = 0x8
- };
- Q_DECLARE_FLAGS(Flags, Flag)
-#endif
-
-protected:
-
- struct Interface
- {
- // ### FIXME: need a QByteArrayView
- using DecoderFn = QChar * (*)(QChar *out, const char *in, qsizetype length, State *state);
- using LengthFn = qsizetype (*)(qsizetype inLength);
- using EncoderFn = char * (*)(char *out, QStringView in, State *state);
- const char *name = nullptr;
- DecoderFn toUtf16 = nullptr;
- LengthFn toUtf16Len = nullptr;
- EncoderFn fromUtf16 = nullptr;
- LengthFn fromUtf16Len = nullptr;
- };
-
- QSTRINGCONVERTER_CONSTEXPR QStringConverter()
- : iface(nullptr)
- {}
- QSTRINGCONVERTER_CONSTEXPR QStringConverter(Encoding encoding, Flags f)
- : iface(&encodingInterfaces[int(encoding)]), state(f)
- {}
- QSTRINGCONVERTER_CONSTEXPR QStringConverter(const Interface *i)
- : iface(i)
- {}
- Q_CORE_EXPORT QStringConverter(const char *name, Flags f);
-
-
-public:
- bool isValid() const { return iface != nullptr; }
-
- void resetState()
- {
- state.clear();
- }
- bool hasError() const { return state.invalidChars != 0; }
-
- const char *name() const
- { return isValid() ? iface->name : nullptr; }
-
- Q_CORE_EXPORT static std::optional<Encoding> encodingForName(const char *name);
- Q_CORE_EXPORT static const char *nameForEncoding(Encoding e);
- Q_CORE_EXPORT static std::optional<Encoding> encodingForData(const char *buf, qsizetype arraySize, char16_t expectedFirstCharacter = 0);
- Q_CORE_EXPORT static std::optional<Encoding> encodingForHtml(const char *buf, qsizetype arraySize);
-
-protected:
- const Interface *iface;
- State state;
-private:
- Q_CORE_EXPORT static const Interface encodingInterfaces[Encoding::LastEncoding + 1];
-};
-
class QStringEncoder : public QStringConverter
{
protected:
- QSTRINGCONVERTER_CONSTEXPR QStringEncoder(const Interface *i)
+ constexpr explicit QStringEncoder(const Interface *i) noexcept
: QStringConverter(i)
{}
public:
- QSTRINGCONVERTER_CONSTEXPR QStringEncoder()
+ constexpr QStringEncoder() noexcept
: QStringConverter()
{}
- QSTRINGCONVERTER_CONSTEXPR QStringEncoder(Encoding encoding, Flags flags = Flag::Default)
+ constexpr explicit QStringEncoder(Encoding encoding, Flags flags = Flag::Default)
: QStringConverter(encoding, flags)
{}
- QStringEncoder(const char *name, Flags flags = Flag::Default)
+ explicit QStringEncoder(const char *name, Flags flags = Flag::Default)
: QStringConverter(name, flags)
{}
-#if defined(Q_QDOC)
- QByteArray operator()(const QString &in);
- QByteArray operator()(QStringView in);
- QByteArray operator()(const QChar *in, qsizetype length);
- QByteArray encode(const QString &in);
- QByteArray encode(QStringView in);
- QByteArray encode(const QChar *in, qsizetype length);
-#else
template<typename T>
struct DecodedData
{
QStringEncoder *encoder;
T data;
- operator QByteArray() const { return encoder->encodeAsByteArray(QStringView(data)); }
+ operator QByteArray() const { return encoder->encodeAsByteArray(data); }
};
+ Q_WEAK_OVERLOAD
DecodedData<const QString &> operator()(const QString &str)
{ return DecodedData<const QString &>{this, str}; }
DecodedData<QStringView> operator()(QStringView in)
{ return DecodedData<QStringView>{this, in}; }
- DecodedData<QStringView> operator()(const QChar *in, qsizetype length)
- { return (*this)(QStringView(in, length)); }
+ Q_WEAK_OVERLOAD
DecodedData<const QString &> encode(const QString &str)
{ return DecodedData<const QString &>{this, str}; }
DecodedData<QStringView> encode(QStringView in)
{ return DecodedData<QStringView>{this, in}; }
- DecodedData<QStringView> encode(const QChar *in, qsizetype length)
- { return (*this)(QStringView(in, length)); }
-#endif
qsizetype requiredSpace(qsizetype inputLength) const
- { return iface->fromUtf16Len(inputLength); }
- char *appendToBuffer(char *out, const QChar *in, qsizetype length)
- { return iface->fromUtf16(out, QStringView(in, length), &state); }
+ { return iface ? iface->fromUtf16Len(inputLength) : 0; }
+ char *appendToBuffer(char *out, QStringView in)
+ {
+ if (!iface) {
+ state.invalidChars = 1;
+ return out;
+ }
+ return iface->fromUtf16(out, in, &state);
+ }
private:
QByteArray encodeAsByteArray(QStringView in)
{
+ if (!iface) {
+ // ensure that hasError returns true
+ state.invalidChars = 1;
+ return {};
+ }
QByteArray result(iface->fromUtf16Len(in.size()), Qt::Uninitialized);
char *out = result.data();
out = iface->fromUtf16(out, in, &state);
@@ -253,68 +81,64 @@ private:
class QStringDecoder : public QStringConverter
{
- struct View {
- const char *ch;
- qsizetype l;
- const char *data() const { return ch; }
- qsizetype length() const { return l; }
- };
-
protected:
- QSTRINGCONVERTER_CONSTEXPR QStringDecoder(const Interface *i)
+ constexpr explicit QStringDecoder(const Interface *i) noexcept
: QStringConverter(i)
{}
public:
- QSTRINGCONVERTER_CONSTEXPR QStringDecoder(Encoding encoding, Flags flags = Flag::Default)
+ constexpr explicit QStringDecoder(Encoding encoding, Flags flags = Flag::Default)
: QStringConverter(encoding, flags)
{}
- QSTRINGCONVERTER_CONSTEXPR QStringDecoder()
+ constexpr QStringDecoder() noexcept
: QStringConverter()
{}
- QStringDecoder(const char *name, Flags f = Flag::Default)
+ explicit QStringDecoder(const char *name, Flags f = Flag::Default)
: QStringConverter(name, f)
{}
-#if defined(Q_QDOC)
- QString operator()(const QByteArray &ba);
- QString operator()(const char *in, qsizetype size);
- QString operator()(const char *chars);
- QString decode(const QByteArray &ba);
- QString decode(const char *in, qsizetype size);
- QString decode(const char *chars);
-#else
template<typename T>
struct EncodedData
{
QStringDecoder *decoder;
T data;
- operator QString() const { return decoder->decodeAsString(data.data(), data.length()); }
+ operator QString() const { return decoder->decodeAsString(data); }
};
+ Q_WEAK_OVERLOAD
EncodedData<const QByteArray &> operator()(const QByteArray &ba)
{ return EncodedData<const QByteArray &>{this, ba}; }
- EncodedData<View> operator()(const char *in, qsizetype length)
- { return EncodedData<View>{this, {in, length}}; }
- EncodedData<View> operator()(const char *chars)
- { return EncodedData<View>{this, {chars, qsizetype(strlen(chars))}}; }
+ EncodedData<QByteArrayView> operator()(QByteArrayView ba)
+ { return EncodedData<QByteArrayView>{this, ba}; }
+ Q_WEAK_OVERLOAD
EncodedData<const QByteArray &> decode(const QByteArray &ba)
{ return EncodedData<const QByteArray &>{this, ba}; }
- EncodedData<View> decode(const char *in, qsizetype length)
- { return EncodedData<View>{this, {in, length}}; }
- EncodedData<View> decode(const char *chars)
- { return EncodedData<View>{this, {chars, qsizetype(strlen(chars))}}; }
-#endif
+ EncodedData<QByteArrayView> decode(QByteArrayView ba)
+ { return EncodedData<QByteArrayView>{this, ba}; }
qsizetype requiredSpace(qsizetype inputLength) const
- { return iface->toUtf16Len(inputLength); }
- QChar *appendToBuffer(QChar *out, const char *in, qsizetype length)
- { return iface->toUtf16(out, in, length, &state); }
+ { return iface ? iface->toUtf16Len(inputLength) : 0; }
+ QChar *appendToBuffer(QChar *out, QByteArrayView ba)
+ {
+ if (!iface) {
+ state.invalidChars = 1;
+ return out;
+ }
+ return iface->toUtf16(out, ba, &state);
+ }
+ char16_t *appendToBuffer(char16_t *out, QByteArrayView ba)
+ { return reinterpret_cast<char16_t *>(appendToBuffer(reinterpret_cast<QChar *>(out), ba)); }
+
+ Q_CORE_EXPORT static QStringDecoder decoderForHtml(QByteArrayView data);
+
private:
- QString decodeAsString(const char *in, qsizetype length)
+ QString decodeAsString(QByteArrayView in)
{
- QString result(iface->toUtf16Len(length), Qt::Uninitialized);
- QChar *out = result.data();
- // ### Fixme: state handling needs to be moved into the conversion methods
- out = iface->toUtf16(out, in, length, &state);
+ if (!iface) {
+ // ensure that hasError returns true
+ state.invalidChars = 1;
+ return {};
+ }
+ QString result(iface->toUtf16Len(in.size()), Qt::Uninitialized);
+ const QChar *out = iface->toUtf16(result.data(), in, &state);
result.truncate(out - result.constData());
return result;
}
@@ -329,10 +153,10 @@ struct QConcatenable<QStringDecoder::EncodedData<T>>
typedef QChar type;
typedef QString ConvertTo;
enum { ExactSize = false };
- static qsizetype size(const QStringDecoder::EncodedData<T> &s) { return s.decoder->requiredSpace(s.data.length()); }
+ static qsizetype size(const QStringDecoder::EncodedData<T> &s) { return s.decoder->requiredSpace(s.data.size()); }
static inline void appendTo(const QStringDecoder::EncodedData<T> &s, QChar *&out)
{
- out = s.decoder->appendToBuffer(out, s.data.data(), s.data.length());
+ out = s.decoder->appendToBuffer(out, s.data);
}
};
@@ -343,10 +167,10 @@ struct QConcatenable<QStringEncoder::DecodedData<T>>
typedef char type;
typedef QByteArray ConvertTo;
enum { ExactSize = false };
- static qsizetype size(const QStringEncoder::DecodedData<T> &s) { return s.encoder->requiredSpace(s.data.length()); }
+ static qsizetype size(const QStringEncoder::DecodedData<T> &s) { return s.encoder->requiredSpace(s.data.size()); }
static inline void appendTo(const QStringEncoder::DecodedData<T> &s, char *&out)
{
- out = s.encoder->appendToBuffer(out, s.data.data(), s.data.length());
+ out = s.encoder->appendToBuffer(out, s.data);
}
};
@@ -373,8 +197,66 @@ QByteArray &operator+=(QByteArray &a, const QStringEncoder::DecodedData<T> &b)
}
#endif
-QT_END_NAMESPACE
+template <typename InputIterator>
+void QString::assign_helper_char8(InputIterator first, InputIterator last)
+{
+ static_assert(!QString::is_contiguous_iterator_v<InputIterator>,
+ "Internal error: Should have been handed over to the QAnyStringView overload."
+ );
+
+ using ValueType = typename std::iterator_traits<InputIterator>::value_type;
+ constexpr bool IsFwdIt = std::is_convertible_v<
+ typename std::iterator_traits<InputIterator>::iterator_category,
+ std::forward_iterator_tag
+ >;
+
+ resize(0);
+ // In case of not being shared, there is the possibility of having free space at begin
+ // even after the resize to zero.
+ if (const auto offset = d.freeSpaceAtBegin())
+ d.setBegin(d.begin() - offset);
+
+ if constexpr (IsFwdIt)
+ reserve(static_cast<qsizetype>(std::distance(first, last)));
+
+ auto toUtf16 = QStringDecoder(QStringDecoder::Utf8);
+ auto availableCapacity = d.constAllocatedCapacity();
+ auto *dst = d.data();
+ auto *dend = d.data() + availableCapacity;
+
+ while (true) {
+ if (first == last) { // ran out of input elements
+ Q_ASSERT(!std::less<>{}(dend, dst));
+ d.size = dst - d.begin();
+ return;
+ }
+ const ValueType next = *first; // decays proxies, if any
+ const auto chunk = QUtf8StringView(&next, 1);
+ // UTF-8 characters can have a maximum size of 4 bytes and may result in a surrogate
+ // pair of UTF-16 code units. In the input-iterator case, we don't know the size
+ // and would need to always reserve space for 2 code units. To keep our promise
+ // of 'not allocating if it fits', we have to pre-check this condition.
+ // We know that it fits in the forward-iterator case.
+ if constexpr (!IsFwdIt) {
+ constexpr qsizetype Pair = 2;
+ char16_t buf[Pair];
+ const qptrdiff n = toUtf16.appendToBuffer(buf, chunk) - buf;
+ if (dend - dst < n) { // ran out of allocated memory
+ const auto offset = dst - d.begin();
+ reallocData(d.constAllocatedCapacity() + Pair, QArrayData::Grow);
+ // update the pointers since we've re-allocated
+ availableCapacity = d.constAllocatedCapacity();
+ dst = d.data() + offset;
+ dend = d.data() + availableCapacity;
+ }
+ dst = std::copy_n(buf, n, dst);
+ } else { // take the fast path
+ dst = toUtf16.appendToBuffer(dst, chunk);
+ }
+ ++first;
+ }
+}
-#undef QSTRINGCONVERTER_CONSTEXPR
+QT_END_NAMESPACE
#endif