Implement support for ref-qualified QString::toLatin1 & friends

This is the first step in implementing an in-place conversion of QString
to QByteArray. This requires ref-qualifiers in member functions so we
know that we have an rvalue QString.

Converting from UTF-16 to Latin1 always requires half the memory.

For conversion from UTF-16 to UTF-8, the typical string will also need
the same memory or less: characters from U+0000 to U+007F consume one
fewer byte; characters from U+0080 to U+07FF and from U+10000 to
U+1FFFFF occupy the same space in UTF-8 and UTF-16; it's only the ones
from U+0800 to U+FFFF that consume more space in the UTF-8 string.

For the locale's 8-bit codec, we can't be sure and the code (currently)
needs to go through QTextCodec anyway.

This requires a #define set before #include'ing "qstring.h". However,
since qstring.h is included by the QtCore PCH, we need an extra qmake
compiler without the PCH flags to compile this .cpp.

After this change, the distribution of calls in QtCore, Network, Gui,
and Widgets is as follows:

                  const &               &&
 toUtf8           31 (74%)              11 (26%)
 toLatin1         79 (77%)              24 (23%)
 toLocal8Bit      26 (16%)              138 (84%)

Change-Id: Idd96f9ddb51b989bc59f6da50054dd10c953dd4f
Reviewed-by: Olivier Goffart <ogoffart@woboq.com>

11 files changed, 141 insertions, 24 deletions

diff --git a/qmake/Makefile.unix b/qmake/Makefile.unix
index 2f04a88a42..2c41bf5951 100644
--- a/qmake/Makefile.unix
+++ b/qmake/Makefile.unix
@@ -12,7 +12,7 @@ OBJS=project.o option.o property.o main.o ioutils.o proitems.o \
      gbuild.o cesdkhandler.o
 
 #qt code
-QOBJS=qtextcodec.o qutfcodec.o qstring.o qstringbuilder.o qtextstream.o qiodevice.o qmalloc.o qglobal.o \
+QOBJS=qtextcodec.o qutfcodec.o qstring.o qstring_compat.o qstringbuilder.o qtextstream.o qiodevice.o qmalloc.o qglobal.o \
       qarraydata.o qbytearray.o qbytearraymatcher.o qdatastream.o qbuffer.o qlist.o qfiledevice.o qfile.o \
       qfilesystementry.o qfilesystemengine.o qfsfileengine.o qfsfileengine_iterator.o qregexp.o qvector.o \
       qbitarray.o qdir.o qdiriterator.o quuid.o qhash.o qfileinfo.o qdatetime.o qstringlist.o \
@@ -40,6 +40,7 @@ DEPEND_SRC = \
 	   $(QMKGENSRC)/win32/msvc_objectmodel.cpp $(QMKGENSRC)/win32/msbuild_objectmodel.cpp \
 	   $(SOURCE_PATH)/src/corelib/codecs/qtextcodec.cpp $(SOURCE_PATH)/src/corelib/codecs/qutfcodec.cpp \
 	   $(SOURCE_PATH)/src/corelib/tools/qstring.cpp $(SOURCE_PATH)/src/corelib/io/qfile.cpp \
+	   $(SOURCE_PATH)/src/corelib/tools/qstring_compat.cpp \
 	   $(SOURCE_PATH)/src/corelib/io/qfiledevice.cpp \
 	   $(SOURCE_PATH)/src/corelib/io/qtextstream.cpp $(SOURCE_PATH)/src/corelib/io/qiodevice.cpp \
 	   $(SOURCE_PATH)/src/corelib/global/qmalloc.cpp \
@@ -272,6 +273,9 @@ qtextcodec.o: $(SOURCE_PATH)/src/corelib/codecs/qtextcodec.cpp
 qstring.o: $(SOURCE_PATH)/src/corelib/tools/qstring.cpp
 	$(CXX) -c -o $@ $(CXXFLAGS) $(SOURCE_PATH)/src/corelib/tools/qstring.cpp
 
+qstring_compat.o: $(SOURCE_PATH)/src/corelib/tools/qstring_compat.cpp
+	$(CXX) -c -o $@ $(CXXFLAGS) $(SOURCE_PATH)/src/corelib/tools/qstring_compat.cpp
+
 qstringbuilder.o: $(SOURCE_PATH)/src/corelib/tools/qstringbuilder.cpp
 	$(CXX) -c -o $@ $(CXXFLAGS) $(SOURCE_PATH)/src/corelib/tools/qstringbuilder.cpp
 
diff --git a/qmake/Makefile.win32 b/qmake/Makefile.win32
index 73f4dd5de3..0bc0093fd3 100644
--- a/qmake/Makefile.win32
+++ b/qmake/Makefile.win32
@@ -105,6 +105,7 @@ QTOBJS= \
 	qtextcodec.obj \
 	qutfcodec.obj \
 	qstring.obj \
+	qstring_compat.obj \
 	qstringlist.obj \
 	qstringbuilder.obj \
 	qsystemerror.obj \
diff --git a/qmake/qmake.pri b/qmake/qmake.pri
index 661b787a55..6052dadb2e 100644
--- a/qmake/qmake.pri
+++ b/qmake/qmake.pri
@@ -67,6 +67,7 @@ bootstrap { #Qt code
         qtextcodec.cpp \
         qutfcodec.cpp \
         qstring.cpp \
+        qstring_compat.cpp \
         qstringlist.cpp \
         qtemporaryfile.cpp \
         qtextstream.cpp \
diff --git a/src/corelib/tools/qstring.cpp b/src/corelib/tools/qstring.cpp
index 77d002e4d5..a94b18b5d3 100644
--- a/src/corelib/tools/qstring.cpp
+++ b/src/corelib/tools/qstring.cpp
@@ -4006,7 +4006,22 @@ static QByteArray toLatin1_helper(const QChar *data, int length)
     return ba;
 }
 
+QByteArray QString::toLatin1_helper(const QString &string)
+{
+    if (Q_UNLIKELY(string.isNull()))
+        return QByteArray();
+
+    return toLatin1_helper(string.constData(), string.length());
+}
+
+QByteArray QString::toLatin1_helper(const QChar *data, int length)
+{
+    return QT_PREPEND_NAMESPACE(toLatin1_helper)(data, length);
+}
+
 /*!
+    \fn QByteArray QString::toLatin1() const
+
     Returns a Latin-1 representation of the string as a QByteArray.
 
     The returned byte array is undefined if the string contains non-Latin1
@@ -4015,10 +4030,6 @@ static QByteArray toLatin1_helper(const QChar *data, int length)
 
     \sa fromLatin1(), toUtf8(), toLocal8Bit(), QTextCodec
 */
-QByteArray QString::toLatin1() const
-{
-    return toLatin1_helper(unicode(), length());
-}
 
 /*!
     \fn QByteArray QString::toAscii() const
@@ -4033,19 +4044,9 @@ QByteArray QString::toLatin1() const
     \sa fromAscii(), toLatin1(), toUtf8(), toLocal8Bit(), QTextCodec
 */
 
-#if !defined(Q_OS_MAC) && defined(Q_OS_UNIX) && !defined(QT_USE_ICU)
-static QByteArray toLocal8Bit_helper(const QChar *data, int length)
-{
-#ifndef QT_NO_TEXTCODEC
-    QTextCodec *localeCodec = QTextCodec::codecForLocale();
-    if (localeCodec)
-        return localeCodec->fromUnicode(data, length);
-#endif // QT_NO_TEXTCODEC
-    return toLatin1_helper(data, length);
-}
-#endif
-
 /*!
+    \fn QByteArray QString::toLocal8Bit() const
+
     Returns the local 8-bit representation of the string as a
     QByteArray. The returned byte array is undefined if the string
     contains characters not supported by the local 8-bit encoding.
@@ -4060,17 +4061,21 @@ static QByteArray toLocal8Bit_helper(const QChar *data, int length)
 
     \sa fromLocal8Bit(), toLatin1(), toUtf8(), QTextCodec
 */
-QByteArray QString::toLocal8Bit() const
+
+QByteArray QString::toLocal8Bit_helper(const QChar *data, int size)
 {
 #ifndef QT_NO_TEXTCODEC
     QTextCodec *localeCodec = QTextCodec::codecForLocale();
     if (localeCodec)
-        return localeCodec->fromUnicode(*this);
+        return localeCodec->fromUnicode(data, size);
 #endif // QT_NO_TEXTCODEC
-    return toLatin1();
+    return toLatin1_helper(data, size);
 }
 
+
 /*!
+    \fn QByteArray QString::toUtf8() const
+
     Returns a UTF-8 representation of the string as a QByteArray.
 
     UTF-8 is a Unicode codec and can represent all characters in a Unicode
@@ -4086,12 +4091,13 @@ QByteArray QString::toLocal8Bit() const
 
     \sa fromUtf8(), toLatin1(), toLocal8Bit(), QTextCodec
 */
-QByteArray QString::toUtf8() const
+
+QByteArray QString::toUtf8_helper(const QString &str)
 {
-    if (isNull())
+    if (str.isNull())
         return QByteArray();
 
-    return QUtf8::convertFromUnicode(constData(), length(), 0);
+    return QUtf8::convertFromUnicode(str.constData(), str.length(), 0);
 }
 
 /*!
@@ -9292,7 +9298,7 @@ static inline bool qt_ends_with(const QChar *haystack, int haystackLen,
 */
 QByteArray QStringRef::toLatin1() const
 {
-    return toLatin1_helper(unicode(), length());
+    return QString::toLatin1_helper(unicode(), length());
 }
 
 /*!
diff --git a/src/corelib/tools/qstring.h b/src/corelib/tools/qstring.h
index 798ff2f3e0..943d7c266e 100644
--- a/src/corelib/tools/qstring.h
+++ b/src/corelib/tools/qstring.h
@@ -477,9 +477,24 @@ public:
 
     const ushort *utf16() const;
 
+#if defined(Q_COMPILER_REF_QUALIFIERS) && !defined(QT_COMPILING_QSTRING_COMPAT_CPP)
+    QByteArray toLatin1() const & Q_REQUIRED_RESULT
+    { return toLatin1_helper(*this); }
+    QByteArray toLatin1() && Q_REQUIRED_RESULT
+    { return toLatin1_helper(reinterpret_cast<const ushort *>(constData()), size()); }
+    QByteArray toUtf8() const & Q_REQUIRED_RESULT
+    { return toUtf8_helper(*this); }
+    QByteArray toUtf8() && Q_REQUIRED_RESULT
+    { return toUtf8_helper(*this); }
+    QByteArray toLocal8Bit() const & Q_REQUIRED_RESULT
+    { return toLocal8Bit_helper(constData(), size()); }
+    QByteArray toLocal8Bit() && Q_REQUIRED_RESULT
+    { return toLocal8Bit_helper(constData(), size()); }
+#else
     QByteArray toLatin1() const Q_REQUIRED_RESULT;
     QByteArray toUtf8() const Q_REQUIRED_RESULT;
     QByteArray toLocal8Bit() const Q_REQUIRED_RESULT;
+#endif
     QVector<uint> toUcs4() const Q_REQUIRED_RESULT;
 
     // note - this are all inline so we can benefit from strlen() compile time optimizations
@@ -734,6 +749,10 @@ private:
     static Data *fromAscii_helper(const char *str, int size = -1);
     static QString fromUtf8_helper(const char *str, int size);
     static QString fromLocal8Bit_helper(const char *, int size);
+    static QByteArray toLatin1_helper(const QString &);
+    static QByteArray toLatin1_helper(const QChar *data, int size);
+    static QByteArray toUtf8_helper(const QString &);
+    static QByteArray toLocal8Bit_helper(const QChar *data, int size);
     static int toUcs4_helper(const ushort *uc, int length, uint *out);
     void replace_helper(uint *indices, int nIndices, int blen, const QChar *after, int alen);
     friend class QCharRef;
diff --git a/src/corelib/tools/qstring_compat.cpp b/src/corelib/tools/qstring_compat.cpp
new file mode 100644
index 0000000000..34c37a9cb8
--- /dev/null
+++ b/src/corelib/tools/qstring_compat.cpp
@@ -0,0 +1,68 @@
+/****************************************************************************
+**
+** Copyright (C) 2013 Intel Corporation
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#if defined(QSTRING_H) && !defined(QT_BOOTSTRAPPED)
+#  error "This file cannot be compiled with pre-compiled headers"
+//       (unless it's configure.exe, which is bootstrapped)
+#endif
+#define QT_COMPILING_QSTRING_COMPAT_CPP
+
+#include "qstring.h"
+
+QT_BEGIN_NAMESPACE
+
+// all these implementations must be the same as the inline versions in qstring.h
+QByteArray QString::toLatin1() const
+{
+    return toLatin1_helper(*this);
+}
+
+QByteArray QString::toLocal8Bit() const
+{
+    return toLocal8Bit_helper(constData(), size());
+}
+
+QByteArray QString::toUtf8() const
+{
+    return toUtf8_helper(*this);
+}
+
+QT_END_NAMESPACE
diff --git a/src/corelib/tools/tools.pri b/src/corelib/tools/tools.pri
index 5de2364fe2..c69d270e33 100644
--- a/src/corelib/tools/tools.pri
+++ b/src/corelib/tools/tools.pri
@@ -114,6 +114,9 @@ SOURCES += \
         tools/qvector.cpp \
         tools/qvsnprintf.cpp
 
+NO_PCH_SOURCES = tools/qstring_compat.cpp
+false: SOURCES += $$NO_PCH_SOURCES
+
 !nacl:mac: {
     SOURCES += tools/qelapsedtimer_mac.cpp
     OBJECTIVE_SOURCES += tools/qlocale_mac.mm \
@@ -195,3 +198,14 @@ INCLUDEPATH += ../3rdparty/md5 \
 !macx-icc:!vxworks:unix:LIBS_PRIVATE += -lm
 
 TR_EXCLUDE += ../3rdparty/*
+
+no_pch_compiler.commands = $$QMAKE_CXX -c $(CXXFLAGS) $(INCPATH) ${QMAKE_FILE_IN}
+win32:!gcc:no_pch_compiler.commands += -Fo${QMAKE_FILE_OUT}
+else:no_pch_compiler.commands += -o ${QMAKE_FILE_OUT}
+no_pch_compiler.dependency_type = TYPE_C
+no_pch_compiler.output = ${QMAKE_VAR_OBJECTS_DIR}${QMAKE_FILE_BASE}$${first(QMAKE_EXT_OBJ)}
+no_pch_compiler.input = NO_PCH_SOURCES
+no_pch_compiler.variable_out = OBJECTS
+no_pch_compiler.name = compiling[no_pch] ${QMAKE_FILE_IN}
+silent:no_pch_compiler.commands = @echo compiling[no_pch] ${QMAKE_FILE_IN} && $$no_pch_compiler.commands
+QMAKE_EXTRA_COMPILERS += no_pch_compiler
diff --git a/src/tools/bootstrap/bootstrap.pro b/src/tools/bootstrap/bootstrap.pro
index dd5189758d..967f45d5df 100644
--- a/src/tools/bootstrap/bootstrap.pro
+++ b/src/tools/bootstrap/bootstrap.pro
@@ -100,6 +100,7 @@ SOURCES += \
            ../../corelib/tools/qsize.cpp \
            ../../corelib/tools/qline.cpp \
            ../../corelib/tools/qstring.cpp \
+           ../../corelib/tools/qstring_compat.cpp \
            ../../corelib/tools/qstringlist.cpp \
            ../../corelib/tools/qvector.cpp \
            ../../corelib/tools/qvsnprintf.cpp \
diff --git a/tools/configure/Makefile.mingw b/tools/configure/Makefile.mingw
index 1994c9a52b..11bfbe562a 100644
--- a/tools/configure/Makefile.mingw
+++ b/tools/configure/Makefile.mingw
@@ -60,6 +60,7 @@ OBJECTS = \
     qmap.o \
     qregexp.o \
     qstring.o \
+    qstring_compat.o \
     qstringlist.o \
     qvsnprintf.o \
     qvariant.o \
diff --git a/tools/configure/Makefile.win32 b/tools/configure/Makefile.win32
index 17c2d1089c..272b86c2d4 100644
--- a/tools/configure/Makefile.win32
+++ b/tools/configure/Makefile.win32
@@ -58,6 +58,7 @@ OBJECTS = \
     qmap.obj \
     qregexp.obj \
     qstring.obj \
+    qstring_compat.obj \
     qstringlist.obj \
     qvsnprintf.obj \
     qvariant.obj \
diff --git a/tools/configure/configure.pro b/tools/configure/configure.pro
index 2163e37a58..215a137adf 100644
--- a/tools/configure/configure.pro
+++ b/tools/configure/configure.pro
@@ -119,6 +119,7 @@ SOURCES  = main.cpp configureapp.cpp environment.cpp tools.cpp \
            $$QT_SOURCE_TREE/src/corelib/tools/qmap.cpp \
            $$QT_SOURCE_TREE/src/corelib/tools/qregexp.cpp \
            $$QT_SOURCE_TREE/src/corelib/tools/qstring.cpp \
+           $$QT_SOURCE_TREE/src/corelib/tools/qstring_compat.cpp \
            $$QT_SOURCE_TREE/src/corelib/tools/qstringlist.cpp \
            $$QT_SOURCE_TREE/src/corelib/tools/qvsnprintf.cpp \
            $$QT_SOURCE_TREE/src/corelib/kernel/qvariant.cpp \