Merge remote-tracking branch 'origin/release' into stable

Change-Id: If1abbe7810ea43ae750db91066f9f579c79b2289

8 files changed, 185 insertions, 35 deletions

diff --git a/mkspecs/features/qt_build_config.prf b/mkspecs/features/qt_build_config.prf
index b611e782f9..0bf90cc297 100644
--- a/mkspecs/features/qt_build_config.prf
+++ b/mkspecs/features/qt_build_config.prf
@@ -34,8 +34,12 @@ QMAKE_DIR_REPLACE_SANE = PRECOMPILED_DIR OBJECTS_DIR MOC_DIR RCC_DIR UI_DIR
 
 !build_pass:!isEmpty(_QMAKE_SUPER_CACHE_):force_independent {
     # When doing a -prefix build of top-level qt5/qt.pro, we need to announce
-    # this repo's module pris' location to the other repos.
-    isEmpty(MODULE_QMAKE_OUTDIR): MODULE_QMAKE_OUTDIR = $$shadowed($$dirname(_QMAKE_CONF_))
+    # this repo's output dir to the other repos.
+    MODULE_BASE_OUTDIR = $$shadowed($$dirname(_QMAKE_CONF_))
+    !contains(QTREPOS, $$MODULE_BASE_OUTDIR): \
+        cache(QTREPOS, add super, MODULE_BASE_OUTDIR)
+    # This repo's module pris' location needs to be made known to qmake.
+    isEmpty(MODULE_QMAKE_OUTDIR): MODULE_QMAKE_OUTDIR = $$MODULE_BASE_OUTDIR
     modpath = $$MODULE_QMAKE_OUTDIR/mkspecs/modules
     !contains(QMAKEMODULES, $$modpath): \
         cache(QMAKEMODULES, add super, modpath)
diff --git a/mkspecs/features/qt_docs.prf b/mkspecs/features/qt_docs.prf
index be3cd5273c..8e63fa61a7 100644
--- a/mkspecs/features/qt_docs.prf
+++ b/mkspecs/features/qt_docs.prf
@@ -23,8 +23,9 @@ QDOC += -outputdir $$QMAKE_DOCS_OUTPUTDIR
 !build_online_docs: \
     QDOC += -installdir $$[QT_INSTALL_DOCS]
 DOC_INDEXES =
-for(qmod, QMAKEMODULES): \
-    DOC_INDEXES += -indexdir $$section(qmod, /, 0, -3)/doc
+for(qrep, QTREPOS): \
+    exists($$qrep/doc): \
+        DOC_INDEXES += -indexdir $$qrep/doc
 qtver.name = QT_VERSION
 qtver.value = $$VERSION
 isEmpty(qtver.value): qtver.value = $$MODULE_VERSION
diff --git a/src/3rdparty/harfbuzz-ng/src/config.h b/src/3rdparty/harfbuzz-ng/src/config.h
index cb68ab0e5b..9abb5df7f1 100644
--- a/src/3rdparty/harfbuzz-ng/src/config.h
+++ b/src/3rdparty/harfbuzz-ng/src/config.h
@@ -4,7 +4,6 @@
 #define HAVE_OT
 #define HAVE_ATEXIT
 
-#define HB_NO_MT
 #define HB_NO_UNICODE_FUNCS
 
 #define HB_DISABLE_DEPRECATED
diff --git a/src/corelib/codecs/qutfcodec.cpp b/src/corelib/codecs/qutfcodec.cpp
index 072cda63aa..c5f580e13d 100644
--- a/src/corelib/codecs/qutfcodec.cpp
+++ b/src/corelib/codecs/qutfcodec.cpp
@@ -237,7 +237,20 @@ QByteArray QUtf8::convertFromUnicode(const QChar *uc, int len, QTextCodec::Conve
 
 QString QUtf8::convertToUnicode(const char *chars, int len)
 {
-    QString result(len + 1, Qt::Uninitialized); // worst case
+    // UTF-8 to UTF-16 always needs the exact same number of words or less:
+    //    UTF-8     UTF-16
+    //   1 byte     1 word
+    //   2 bytes    1 word
+    //   3 bytes    1 word
+    //   4 bytes    2 words (one surrogate pair)
+    // That is, we'll use the full buffer if the input is US-ASCII (1-byte UTF-8),
+    // half the buffer for U+0080-U+07FF text (e.g., Greek, Cyrillic, Arabic) or
+    // non-BMP text, and one third of the buffer for U+0800-U+FFFF text (e.g, CJK).
+    //
+    // The table holds for invalid sequences too: we'll insert one replacement char
+    // per invalid byte.
+    QString result(len, Qt::Uninitialized);
+
     ushort *dst = reinterpret_cast<ushort *>(const_cast<QChar *>(result.constData()));
     const uchar *src = reinterpret_cast<const uchar *>(chars);
     const uchar *end = src + len;
@@ -282,7 +295,18 @@ QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::Converte
     int res;
     uchar ch = 0;
 
-    QString result(need + len + 1, Qt::Uninitialized); // worst case
+    // See above for buffer requirements for stateless decoding. However, that
+    // fails if the state is not empty. The following situations can add to the
+    // requirements:
+    //  state contains      chars starts with           requirement
+    //   1 of 2 bytes       valid continuation          0
+    //   2 of 3 bytes       same                        0
+    //   3 bytes of 4       same                        +1 (need to insert surrogate pair)
+    //   1 of 2 bytes       invalid continuation        +1 (need to insert replacement and restart)
+    //   2 of 3 bytes       same                        +1 (same)
+    //   3 of 4 bytes       same                        +1 (same)
+    QString result(need + len + 1, Qt::Uninitialized);
+
     ushort *dst = reinterpret_cast<ushort *>(const_cast<QChar *>(result.constData()));
     const uchar *src = reinterpret_cast<const uchar *>(chars);
     const uchar *end = src + len;
@@ -305,15 +329,17 @@ QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::Converte
             const uchar *begin = &remainingCharsData[1];
             res = QUtf8Functions::fromUtf8<QUtf8BaseTraits>(remainingCharsData[0], dst, begin,
                     static_cast<const uchar *>(remainingCharsData) + remainingCharsCount + newCharsToCopy);
-            if (res == QUtf8BaseTraits::EndOfString) {
+            if (res == QUtf8BaseTraits::Error || (res == QUtf8BaseTraits::EndOfString && len == 0)) {
+                // special case for len == 0:
+                // if we were supplied an empty string, terminate the previous, unfinished sequence with error
+                ++invalid;
+                *dst++ = replacement;
+            } else if (res == QUtf8BaseTraits::EndOfString) {
                 // if we got EndOfString again, then there were too few bytes in src;
                 // copy to our state and return
                 state->remainingChars = remainingCharsCount + newCharsToCopy;
                 memcpy(&state->state_data[0], remainingCharsData, state->remainingChars);
                 return QString();
-            } else if (res == QUtf8BaseTraits::Error) {
-                ++invalid;
-                *dst++ = replacement;
             } else if (!headerdone && res >= 0) {
                 // eat the UTF-8 BOM
                 headerdone = true;
@@ -322,8 +348,10 @@ QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::Converte
             }
 
             // adjust src now that we have maybe consumed a few chars
-            //Q_ASSERT(res > remainingCharsCount)
-            src += res - remainingCharsCount;
+            if (res >= 0) {
+                Q_ASSERT(res > remainingCharsCount);
+                src += res - remainingCharsCount;
+            }
         }
     }
 
diff --git a/src/corelib/tools/qhash.cpp b/src/corelib/tools/qhash.cpp
index a5d14a3535..ca645636e4 100644
--- a/src/corelib/tools/qhash.cpp
+++ b/src/corelib/tools/qhash.cpp
@@ -125,7 +125,7 @@ static uint crc32(const Char *ptr, size_t len, uint h)
 #  else
     p += 4;
     for ( ; p <= e; p += 4)
-        h = _mm_crc32_u32(h, *reinterpret_cast<const uint *>(p));
+        h = _mm_crc32_u32(h, *reinterpret_cast<const uint *>(p - 4));
     p -= 4;
     len = e - p;
 #  endif
diff --git a/src/network/doc/src/ssl.qdoc b/src/network/doc/src/ssl.qdoc
index e1bb1b9316..77fbec9943 100644
--- a/src/network/doc/src/ssl.qdoc
+++ b/src/network/doc/src/ssl.qdoc
@@ -57,7 +57,7 @@
     system:
 
     \code
-        ./configure -openssl-linked OPENSSL_LIBS='-L/opt/ssl/lib -lssl -lcrypto'
+        OPENSSL_LIBS='-L/opt/ssl/lib -lssl -lcrypto' ./configure -openssl-linked
     \endcode
 
     To disable SSL support in a Qt build, configure Qt with the \c{-no-openssl}
diff --git a/src/plugins/platforms/xcb/qxcbconnection_xi2.cpp b/src/plugins/platforms/xcb/qxcbconnection_xi2.cpp
index efa1691780..eb7b220c43 100644
--- a/src/plugins/platforms/xcb/qxcbconnection_xi2.cpp
+++ b/src/plugins/platforms/xcb/qxcbconnection_xi2.cpp
@@ -148,15 +148,17 @@ void QXcbConnection::initializeXInput2()
                     }
                     case XIButtonClass: {
                         XIButtonClassInfo *bci = reinterpret_cast<XIButtonClassInfo *>(devices[i].classes[c]);
-                        for (int i=0; i < bci->num_buttons; ++i) {
-                            const int buttonAtom = qatom(bci->labels[i]);
-                            if (buttonAtom == QXcbAtom::ButtonWheelUp
-                                || buttonAtom == QXcbAtom::ButtonWheelDown) {
+                        if (bci->num_buttons >= 5) {
+                            Atom label4 = bci->labels[3];
+                            Atom label5 = bci->labels[4];
+                            if ((!label4 || qatom(label4) == QXcbAtom::ButtonWheelUp) && (!label5 || qatom(label5) == QXcbAtom::ButtonWheelDown))
                                 scrollingDevice.legacyOrientations |= Qt::Vertical;
-                            } else if (buttonAtom == QXcbAtom::ButtonHorizWheelLeft
-                                       || buttonAtom == QXcbAtom::ButtonHorizWheelRight) {
+                        }
+                        if (bci->num_buttons >= 7) {
+                            Atom label6 = bci->labels[5];
+                            Atom label7 = bci->labels[6];
+                            if ((!label6 || qatom(label6) == QXcbAtom::ButtonHorizWheelLeft) && (!label7 || qatom(label7) == QXcbAtom::ButtonHorizWheelRight))
                                 scrollingDevice.legacyOrientations |= Qt::Horizontal;
-                            }
                         }
                         break;
                     }
@@ -246,6 +248,7 @@ void QXcbConnection::xi2Select(xcb_window_t window)
     }
 #endif // XCB_USE_XINPUT22
 
+    QSet<int> tabletDevices;
 #ifndef QT_NO_TABLETEVENT
     // For each tablet, select some additional event types.
     // Press, motion, etc. events must never be selected for _all_ devices
@@ -253,15 +256,19 @@ void QXcbConnection::xi2Select(xcb_window_t window)
     // similar handlers useless and we have no intention to infect
     // all the pure xcb code with Xlib-based XI2.
     if (!m_tabletData.isEmpty()) {
+        unsigned int tabletBitMask = bitMask;
+        unsigned char *xiTabletBitMask = reinterpret_cast<unsigned char *>(&tabletBitMask);
         QVector<XIEventMask> xiEventMask(m_tabletData.count());
-        bitMask |= XI_ButtonPressMask;
-        bitMask |= XI_ButtonReleaseMask;
-        bitMask |= XI_MotionMask;
-        bitMask |= XI_PropertyEventMask;
+        tabletBitMask |= XI_ButtonPressMask;
+        tabletBitMask |= XI_ButtonReleaseMask;
+        tabletBitMask |= XI_MotionMask;
+        tabletBitMask |= XI_PropertyEventMask;
         for (int i = 0; i < m_tabletData.count(); ++i) {
-            xiEventMask[i].deviceid = m_tabletData.at(i).deviceId;
-            xiEventMask[i].mask_len = sizeof(bitMask);
-            xiEventMask[i].mask = xiBitMask;
+            int deviceId = m_tabletData.at(i).deviceId;
+            tabletDevices.insert(deviceId);
+            xiEventMask[i].deviceid = deviceId;
+            xiEventMask[i].mask_len = sizeof(tabletBitMask);
+            xiEventMask[i].mask = xiTabletBitMask;
         }
         XISelectEvents(xDisplay, window, xiEventMask.data(), m_tabletData.count());
     }
@@ -271,17 +278,30 @@ void QXcbConnection::xi2Select(xcb_window_t window)
     // Enable each scroll device
     if (!m_scrollingDevices.isEmpty()) {
         QVector<XIEventMask> xiEventMask(m_scrollingDevices.size());
-        bitMask = XI_MotionMask;
+        unsigned int scrollBitMask = 0;
+        unsigned char *xiScrollBitMask = reinterpret_cast<unsigned char *>(&scrollBitMask);
+        scrollBitMask = XI_MotionMask;
+        scrollBitMask |= XI_ButtonReleaseMask;
+        bitMask |= XI_MotionMask;
         bitMask |= XI_ButtonReleaseMask;
         int i=0;
         Q_FOREACH (const ScrollingDevice& scrollingDevice, m_scrollingDevices) {
+            if (tabletDevices.contains(scrollingDevice.deviceId))
+                continue; // All necessary events are already captured.
             xiEventMask[i].deviceid = scrollingDevice.deviceId;
-            xiEventMask[i].mask_len = sizeof(bitMask);
-            xiEventMask[i].mask = xiBitMask;
+            if (m_touchDevices.contains(scrollingDevice.deviceId)) {
+                xiEventMask[i].mask_len = sizeof(bitMask);
+                xiEventMask[i].mask = xiBitMask;
+            } else {
+                xiEventMask[i].mask_len = sizeof(scrollBitMask);
+                xiEventMask[i].mask = xiScrollBitMask;
+            }
             i++;
         }
-        XISelectEvents(xDisplay, window, xiEventMask.data(), m_scrollingDevices.size());
+        XISelectEvents(xDisplay, window, xiEventMask.data(), i);
     }
+#else
+    Q_UNUSED(xiBitMask);
 #endif
 }
 
@@ -655,13 +675,15 @@ bool QXcbConnection::xi2HandleTabletEvent(void *event, TabletData *tabletData)
         if (reinterpret_cast<xXIDeviceEvent *>(event)->detail == 1) { // ignore the physical buttons on the stylus
             tabletData->down = true;
             xi2ReportTabletEvent(*tabletData, xiEvent);
-        }
+        } else
+            handled = false;
         break;
     case XI_ButtonRelease: // stylus up
         if (reinterpret_cast<xXIDeviceEvent *>(event)->detail == 1) {
             tabletData->down = false;
             xi2ReportTabletEvent(*tabletData, xiEvent);
-        }
+        } else
+            handled = false;
         break;
     case XI_Motion:
         // Report TabletMove only when the stylus is touching the tablet.
diff --git a/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp b/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp
index 12b81ee7d4..54e8f8c386 100644
--- a/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp
+++ b/tests/auto/corelib/codecs/qtextcodec/tst_qtextcodec.cpp
@@ -80,6 +80,9 @@ private slots:
     void utf8bom_data();
     void utf8bom();
 
+    void utf8stateful_data();
+    void utf8stateful();
+
     void utfHeaders_data();
     void utfHeaders();
 
@@ -1611,6 +1614,99 @@ void tst_QTextCodec::utf8bom()
     QCOMPARE(codec->toUnicode(data.constData(), data.length(), &state), result);
 }
 
+void tst_QTextCodec::utf8stateful_data()
+{
+    QTest::addColumn<QByteArray>("buffer1");
+    QTest::addColumn<QByteArray>("buffer2");
+    QTest::addColumn<QString>("result");    // null QString indicates decoder error
+
+    // valid buffer continuations
+    QTest::newRow("1of2+valid") << QByteArray("\xc2") << QByteArray("\xa0") << "\xc2\xa0";
+    QTest::newRow("1of3+valid") << QByteArray("\xe0") << QByteArray("\xa0\x80") << "\xe0\xa0\x80";
+    QTest::newRow("2of3+valid") << QByteArray("\xe0\xa0") << QByteArray("\x80") << "\xe0\xa0\x80";
+    QTest::newRow("1of4+valid") << QByteArray("\360") << QByteArray("\220\210\203") << "\360\220\210\203";
+    QTest::newRow("2of4+valid") << QByteArray("\360\220") << QByteArray("\210\203") << "\360\220\210\203";
+    QTest::newRow("3of4+valid") << QByteArray("\360\220\210") << QByteArray("\203") << "\360\220\210\203";
+    QTest::newRow("1ofBom+valid") << QByteArray("\xef") << QByteArray("\xbb\xbf") << "";
+    QTest::newRow("2ofBom+valid") << QByteArray("\xef\xbb") << QByteArray("\xbf") << "";
+
+    // invalid continuation
+    QTest::newRow("1of2+invalid") << QByteArray("\xc2") << QByteArray("a") << QString();
+    QTest::newRow("1of3+invalid") << QByteArray("\xe0") << QByteArray("a") << QString();
+    QTest::newRow("2of3+invalid") << QByteArray("\xe0\xa0") << QByteArray("a") << QString();
+    QTest::newRow("1of4+invalid") << QByteArray("\360") << QByteArray("a") << QString();
+    QTest::newRow("2of4+invalid") << QByteArray("\360\220") << QByteArray("a") << QString();
+    QTest::newRow("3of4+invalid") << QByteArray("\360\220\210") << QByteArray("a") << QString();
+
+    // invalid: sequence too short (the empty second buffer causes a state reset)
+    QTest::newRow("1of2+empty") << QByteArray("\xc2") << QByteArray() << QString();
+    QTest::newRow("1of3+empty") << QByteArray("\xe0") << QByteArray() << QString();
+    QTest::newRow("2of3+empty") << QByteArray("\xe0\xa0") << QByteArray() << QString();
+    QTest::newRow("1of4+empty") << QByteArray("\360") << QByteArray() << QString();
+    QTest::newRow("2of4+empty") << QByteArray("\360\220") << QByteArray() << QString();
+    QTest::newRow("3of4+empty") << QByteArray("\360\220\210") << QByteArray() << QString();
+
+    // overlong sequence:
+    QTest::newRow("overlong-1of2") << QByteArray("\xc1") << QByteArray("\x81") << QString();
+    QTest::newRow("overlong-1of3") << QByteArray("\xe0") << QByteArray("\x81\x81") << QString();
+    QTest::newRow("overlong-2of3") << QByteArray("\xe0\x81") << QByteArray("\x81") << QString();
+    QTest::newRow("overlong-1of4") << QByteArray("\xf0") << QByteArray("\x80\x81\x81") << QString();
+    QTest::newRow("overlong-2of4") << QByteArray("\xf0\x80") << QByteArray("\x81\x81") << QString();
+    QTest::newRow("overlong-3of4") << QByteArray("\xf0\x80\x81") << QByteArray("\x81") << QString();
+
+    // out of range:
+    // leading byte 0xF4 can produce codepoints above U+10FFFF, which aren't valid
+    QTest::newRow("outofrange1-1of4") << QByteArray("\xf4") << QByteArray("\x90\x80\x80") << QString();
+    QTest::newRow("outofrange1-2of4") << QByteArray("\xf4\x90") << QByteArray("\x80\x80") << QString();
+    QTest::newRow("outofrange1-3of4") << QByteArray("\xf4\x90\x80") << QByteArray("\x80") << QString();
+    QTest::newRow("outofrange2-1of4") << QByteArray("\xf5") << QByteArray("\x90\x80\x80") << QString();
+    QTest::newRow("outofrange2-2of4") << QByteArray("\xf5\x90") << QByteArray("\x80\x80") << QString();
+    QTest::newRow("outofrange2-3of4") << QByteArray("\xf5\x90\x80") << QByteArray("\x80") << QString();
+    QTest::newRow("outofrange-1of5") << QByteArray("\xf8") << QByteArray("\x88\x80\x80\x80") << QString();
+    QTest::newRow("outofrange-2of5") << QByteArray("\xf8\x88") << QByteArray("\x80\x80\x80") << QString();
+    QTest::newRow("outofrange-3of5") << QByteArray("\xf8\x88\x80") << QByteArray("\x80\x80") << QString();
+    QTest::newRow("outofrange-4of5") << QByteArray("\xf8\x88\x80\x80") << QByteArray("\x80") << QString();
+    QTest::newRow("outofrange-1of6") << QByteArray("\xfc") << QByteArray("\x84\x80\x80\x80\x80") << QString();
+    QTest::newRow("outofrange-2of6") << QByteArray("\xfc\x84") << QByteArray("\x80\x80\x80\x80") << QString();
+    QTest::newRow("outofrange-3of6") << QByteArray("\xfc\x84\x80") << QByteArray("\x80\x80\x80") << QString();
+    QTest::newRow("outofrange-4of6") << QByteArray("\xfc\x84\x80\x80") << QByteArray("\x80\x80") << QString();
+    QTest::newRow("outofrange-5of6") << QByteArray("\xfc\x84\x80\x80\x80") << QByteArray("\x80") << QString();
+}
+
+void tst_QTextCodec::utf8stateful()
+{
+    QFETCH(QByteArray, buffer1);
+    QFETCH(QByteArray, buffer2);
+    QFETCH(QString, result);
+
+    QTextCodec *utf8codec = QTextCodec::codecForName("utf-8");
+    QVERIFY(utf8codec);
+
+    QTextCodec::ConverterState state;
+    memset(&state, 0, sizeof state);
+
+    QString decoded1 = utf8codec->toUnicode(buffer1, buffer1.size(), &state);
+    if (result.isNull()) {
+        // the decoder may have found an early error (invalidChars > 0):
+        // if it has, remainingChars == 0;
+        // if it hasn't, then it must have a state
+        QVERIFY2((state.remainingChars == 0) != (state.invalidChars == 0),
+                 "remainingChars = " + QByteArray::number(state.remainingChars) +
+                 "; invalidChars = " + QByteArray::number(state.invalidChars));
+    } else {
+        QVERIFY(state.remainingChars > 0);
+        QCOMPARE(state.invalidChars, 0);
+    }
+
+    QString decoded2 = utf8codec->toUnicode(buffer2, buffer2.size(), &state);
+    QCOMPARE(state.remainingChars, 0);
+    if (result.isNull()) {
+        QVERIFY(state.invalidChars > 0);
+    } else {
+        QCOMPARE(decoded1 + decoded2, result);
+    }
+}
+
 void tst_QTextCodec::utfHeaders_data()
 {
     QTest::addColumn<QByteArray>("codecName");