Ensure that Nbsp characters are preserved by escaping them

Characters that are considered spaces, but aren't just ' ' should always
be escaped so that the exact type of space that they are is preserved.
Since QTextDocument::toPlainText() will convert QChar::Nbsp to ' ', we
need to get the plain text from the handler directly and do our own
conversion where applicable to stop it from replacing the Nbsp characters.

Task-number: QTBUG-35652
Change-Id: I373edca92253ca5f978ba656c8ebbcaf77965e61
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>

5 files changed, 40 insertions, 4 deletions

diff --git a/src/linguist/linguist/linguist.pro b/src/linguist/linguist/linguist.pro
index 7b02bc324..af74df8ec 100644
--- a/src/linguist/linguist/linguist.pro
+++ b/src/linguist/linguist/linguist.pro
@@ -1,4 +1,4 @@
-QT += core-private widgets xml uitools-private printsupport
+QT += core-private gui-private widgets xml uitools-private printsupport
 
 DEFINES += QT_NO_CAST_FROM_ASCII QT_NO_CAST_TO_ASCII
 
diff --git a/src/linguist/linguist/messageeditorwidgets.cpp b/src/linguist/linguist/messageeditorwidgets.cpp
index 03f84a19a..658b8a0fe 100644
--- a/src/linguist/linguist/messageeditorwidgets.cpp
+++ b/src/linguist/linguist/messageeditorwidgets.cpp
@@ -45,6 +45,7 @@
 #include <QTextDocumentFragment>
 #include <QToolButton>
 #include <QVBoxLayout>
+#include <QtGui/private/qtextdocument_p.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -390,13 +391,34 @@ void FormMultiWidget::setTranslation(const QString &text, bool userAction)
         setHidden(text.isEmpty());
 }
 
+// Copied from QTextDocument::toPlainText() and modified to
+// not replace QChar::Nbsp with QLatin1Char(' ')
+QString toPlainText(const QString &text)
+{
+    QString txt = text;
+    QChar *uc = txt.data();
+    QChar *e = uc + txt.size();
+
+    for (; uc != e; ++uc) {
+        switch (uc->unicode()) {
+        case 0xfdd0: // QTextBeginningOfFrame
+        case 0xfdd1: // QTextEndOfFrame
+        case QChar::ParagraphSeparator:
+        case QChar::LineSeparator:
+            *uc = QLatin1Char('\n');
+            break;
+        }
+    }
+    return txt;
+}
+
 QString FormMultiWidget::getTranslation() const
 {
     QString ret;
     for (int i = 0; i < m_editors.count(); ++i) {
         if (i)
             ret += QChar(Translator::BinaryVariantSeparator);
-        ret += m_editors.at(i)->toPlainText();
+        ret += toPlainText(m_editors.at(i)->document()->docHandle()->plainText());
     }
     return ret;
 }
diff --git a/src/linguist/shared/ts.cpp b/src/linguist/shared/ts.cpp
index 5c8bcc129..5914319c9 100644
--- a/src/linguist/shared/ts.cpp
+++ b/src/linguist/shared/ts.cpp
@@ -445,7 +445,8 @@ static QString protect(const QString &str)
     QString result;
     result.reserve(str.length() * 12 / 10);
     for (int i = 0; i != str.size(); ++i) {
-        uint c = str.at(i).unicode();
+        const QChar ch = str[i];
+        uint c = ch.unicode();
         switch (c) {
         case '\"':
             result += QLatin1String("&quot;");
@@ -463,7 +464,7 @@ static QString protect(const QString &str)
             result += QLatin1String("&apos;");
             break;
         default:
-            if (c < 0x20 && c != '\r' && c != '\n' && c != '\t')
+            if ((c < 0x20 || (ch > 0x7f && ch.isSpace())) && c != '\r' && c != '\n' && c != '\t')
                 result += numericEntity(c);
             else // this also covers surrogates
                 result += QChar(c);
diff --git a/tests/auto/linguist/lconvert/data/whitespace.ts b/tests/auto/linguist/lconvert/data/whitespace.ts
new file mode 100644
index 000000000..560fe5d03
--- /dev/null
+++ b/tests/auto/linguist/lconvert/data/whitespace.ts
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.1">
+<context>
+    <name>QObject</name>
+    <message>
+        <source>This is a test</source>
+        <translation>SPACE AA AA, 
+NBSP	 AA&#xa0;AA</translation>
+    </message>
+</context>
+</TS>
diff --git a/tests/auto/linguist/lconvert/tst_lconvert.cpp b/tests/auto/linguist/lconvert/tst_lconvert.cpp
index 2ff8ea508..7f3c60f51 100644
--- a/tests/auto/linguist/lconvert/tst_lconvert.cpp
+++ b/tests/auto/linguist/lconvert/tst_lconvert.cpp
@@ -313,6 +313,7 @@ void tst_lconvert::roundtrips_data()
     QTest::newRow("ts-xliff-ts (msgid)") << "msgid.ts" << tsXlfTs << noArgs;
 
     QTest::newRow("ts-po-ts (endless loop)") << "endless-po-loop.ts" << tsPoTs << noArgs;
+    QTest::newRow("ts-qm-ts (whitespace)") << "whitespace.ts" << tsQmTs << noArgs;
 }
 
 void tst_lconvert::roundtrips()