properly deal with messages which appear in multiple encodings

in ts 1.1 and qm files, messages appear in their native encoding. that
means that a message can appear multiple times - once in utf8 and once
in the codecForTr. however, in ts 2.0 files, everything is utf8 and
messages can have a utf8 flag for the later transformation into qm.
unfortunately, there was no flag to mark that the message is needed in
*both* encodings, and the respective case was completely ignored when
reading ts 1.1 and qm files (causing error messages).

Task-number: 249022
AutoTest: 322690

6 files changed, 299 insertions, 220 deletions

diff --git a/tools/linguist/shared/qm.cpp b/tools/linguist/shared/qm.cpp
index 4627b5d927..8a3658c7c0 100644
--- a/tools/linguist/shared/qm.cpp
+++ b/tools/linguist/shared/qm.cpp
@@ -103,6 +103,43 @@ static uint elfHash(const QByteArray &ba)
     return h;
 }
 
+class ByteTranslatorMessage
+{
+public:
+    ByteTranslatorMessage(
+            const QByteArray &context,
+            const QByteArray &sourceText,
+            const QByteArray &comment,
+            const QStringList &translations) :
+        m_context(context),
+        m_sourcetext(sourceText),
+        m_comment(comment),
+        m_translations(translations)
+    {}
+    const QByteArray &context() const { return m_context; }
+    const QByteArray &sourceText() const { return m_sourcetext; }
+    const QByteArray &comment() const { return m_comment; }
+    const QStringList &translations() const { return m_translations; }
+    bool operator<(const ByteTranslatorMessage& m) const;
+
+private:
+    QByteArray m_context;
+    QByteArray m_sourcetext;
+    QByteArray m_comment;
+    QStringList m_translations;
+};
+
+Q_DECLARE_TYPEINFO(ByteTranslatorMessage, Q_MOVABLE_TYPE);
+
+bool ByteTranslatorMessage::operator<(const ByteTranslatorMessage& m) const
+{
+    if (m_context != m.m_context)
+        return m_context < m.m_context;
+    if (m_sourcetext != m.m_sourcetext)
+        return m_sourcetext < m.m_sourcetext;
+    return m_comment < m.m_comment;
+}
+
 class Releaser
 {
 public:
@@ -133,27 +170,12 @@ public:
         m_codec = QTextCodec::codecForName(codecName);
     }
 
-    TranslatorMessage findMessage(const QString &context,
-        const QString &sourceText, const QString &comment,
-        const QString &fileName = QString(), int lineNumber = -1) const;
-
     bool save(QIODevice *iod);
 
-    void insert(const TranslatorMessage &);
-    void remove(const TranslatorMessage &);
-
-    bool contains(const QString &context, const QString &sourceText,
-        const QString & comment) const;
-
-    bool contains(const QString &context, const QString &comment,
-        const QString &fileName, int lineNumber) const;
+    void insert(const TranslatorMessage &msg, bool forceComment);
 
     void squeeze(TranslatorSaveMode mode);
 
-    QList<TranslatorMessage> messages() const;
-
-    bool isEmpty() const;
-
     void setNumerusRules(const QByteArray &rules);
 
 private:
@@ -163,18 +185,20 @@ private:
     // on turn should be the same as passed to the actual tr(...) calls
     QByteArray originalBytes(const QString &str, bool isUtf8) const;
 
-    Prefix commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const;
+    void insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8);
 
-    uint msgHash(const TranslatorMessage &msg) const;
+    Prefix commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const;
 
-    void writeMessage(const TranslatorMessage & msg, QDataStream & stream,
+    uint msgHash(const ByteTranslatorMessage &msg) const;
+
+    void writeMessage(const ByteTranslatorMessage & msg, QDataStream & stream,
         TranslatorSaveMode strip, Prefix prefix) const;
 
     // for squeezed but non-file data, this is what needs to be deleted
     QByteArray m_messageArray;
     QByteArray m_offsetArray;
     QByteArray m_contextArray;
-    QMap<TranslatorMessage, void *> m_messages;
+    QMap<ByteTranslatorMessage, void *> m_messages;
     QByteArray m_numerusRules;
 
     // Used to reproduce the original bytes
@@ -193,12 +217,12 @@ QByteArray Releaser::originalBytes(const QString &str, bool isUtf8) const
     return m_codec ? m_codec->fromUnicode(str) : str.toLatin1();
 }
 
-uint Releaser::msgHash(const TranslatorMessage &msg) const
+uint Releaser::msgHash(const ByteTranslatorMessage &msg) const
 {
-    return elfHash(originalBytes(msg.sourceText() + msg.comment(), msg.isUtf8()));
+    return elfHash(msg.sourceText() + msg.comment());
 }
 
-Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessage &m2) const
+Prefix Releaser::commonPrefix(const ByteTranslatorMessage &m1, const ByteTranslatorMessage &m2) const
 {
     if (msgHash(m1) != msgHash(m2))
         return NoPrefix;
@@ -211,7 +235,7 @@ Prefix Releaser::commonPrefix(const TranslatorMessage &m1, const TranslatorMessa
     return HashContextSourceTextComment;
 }
 
-void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream,
+void Releaser::writeMessage(const ByteTranslatorMessage &msg, QDataStream &stream,
     TranslatorSaveMode mode, Prefix prefix) const
 {
     for (int i = 0; i < msg.translations().count(); ++i) {
@@ -228,14 +252,14 @@ void Releaser::writeMessage(const TranslatorMessage & msg, QDataStream & stream,
     switch (prefix) {
     default:
     case HashContextSourceTextComment:
-        stream << quint8(Tag_Comment) << originalBytes(msg.comment(), msg.isUtf8());
+        stream << quint8(Tag_Comment) << msg.comment();
         // fall through
     case HashContextSourceText:
-        stream << quint8(Tag_SourceText) << originalBytes(msg.sourceText(), msg.isUtf8());
+        stream << quint8(Tag_SourceText) << msg.sourceText();
         // fall through
     case HashContext:
-        stream << quint8(Tag_Context) << originalBytes(msg.context(), msg.isUtf8());
-        ;
+        stream << quint8(Tag_Context) << msg.context();
+        break;
     }
 
     stream << quint8(Tag_End);
@@ -275,7 +299,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     if (m_messages.isEmpty() && mode == SaveEverything)
         return;
 
-    QMap<TranslatorMessage, void *> messages = m_messages;
+    QMap<ByteTranslatorMessage, void *> messages = m_messages;
 
     // re-build contents
     m_messageArray.clear();
@@ -286,7 +310,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     QMap<Offset, void *> offsets;
 
     QDataStream ms(&m_messageArray, QIODevice::WriteOnly);
-    QMap<TranslatorMessage, void *>::const_iterator it, next;
+    QMap<ByteTranslatorMessage, void *>::const_iterator it, next;
     int cpPrev = 0, cpNext = 0;
     for (it = messages.constBegin(); it != messages.constEnd(); ++it) {
         cpPrev = cpNext;
@@ -310,7 +334,7 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     }
 
     if (mode == SaveStripped) {
-        QMap<QString, int> contextSet;
+        QMap<QByteArray, int> contextSet;
         for (it = messages.constBegin(); it != messages.constEnd(); ++it)
             ++contextSet[it.key().context()];
 
@@ -322,10 +346,10 @@ void Releaser::squeeze(TranslatorSaveMode mode)
         else
             hTableSize = (contextSet.size() < 10000) ? 15013 : 3 * contextSet.size() / 2;
 
-        QMultiMap<int, QString> hashMap;
-        QMap<QString, int>::const_iterator c;
+        QMultiMap<int, QByteArray> hashMap;
+        QMap<QByteArray, int>::const_iterator c;
         for (c = contextSet.constBegin(); c != contextSet.constEnd(); ++c)
-            hashMap.insert(elfHash(originalBytes(c.key(), false /*FIXME*/)) % hTableSize, c.key());
+            hashMap.insert(elfHash(c.key()) % hTableSize, c.key());
 
         /*
           The contexts found in this translator are stored in a hash
@@ -360,16 +384,14 @@ void Releaser::squeeze(TranslatorSaveMode mode)
         t << quint16(0); // the entry at offset 0 cannot be used
         uint upto = 2;
 
-        QMap<int, QString>::const_iterator entry = hashMap.constBegin();
+        QMap<int, QByteArray>::const_iterator entry = hashMap.constBegin();
         while (entry != hashMap.constEnd()) {
             int i = entry.key();
             hTable[i] = quint16(upto >> 1);
 
             do {
-                QString context = entry.value();
-                QByteArray ba = context.toUtf8();
-                const char *con = ba.data();
-                uint len = uint(qstrlen(con));
+                const char *con = entry.value().constData();
+                uint len = uint(entry.value().length());
                 len = qMin(len, 255u);
                 t << quint8(len);
                 t.writeRawData(con, len);
@@ -394,68 +416,28 @@ void Releaser::squeeze(TranslatorSaveMode mode)
     }
 }
 
-bool Releaser::contains(const QString &context, const QString &sourceText,
-    const QString &comment) const
-{
-    return !findMessage(context, sourceText, comment).translation().isNull();
-}
-
-bool Releaser::contains(const QString &context, const QString &comment,
-    const QString &fileName, int lineNumber) const
-{
-    return !findMessage(context, QString(), comment, fileName, lineNumber).isNull();
-}
-
-void Releaser::insert(const TranslatorMessage &message)
-{
-    m_messages.insert(message, 0);
-}
-
-void Releaser::remove(const TranslatorMessage &message)
-{
-    m_messages.remove(message);
-}
-
-
-TranslatorMessage Releaser::findMessage(const QString &context,
-    const QString &sourceText, const QString &comment,
-    const QString &fileName, int lineNumber) const
+void Releaser::insertInternal(const TranslatorMessage &message, bool forceComment, bool isUtf8)
 {
-    if (m_messages.isEmpty()) 
-        return TranslatorMessage();
-
-    QMap<TranslatorMessage, void *>::const_iterator it;
-
-    // Either we want to find an item that matches context, sourcetext
-    // (and optionally comment) Or we want to find an item that
-    // matches context, filename, linenumber (and optionally comment)
-    TranslatorMessage msg(context, sourceText, comment, QString(), fileName, lineNumber);
-    it = m_messages.constFind(msg);
-    if (it != m_messages.constEnd())
-        return it.key();
-
-    if (!comment.isEmpty()) {
-        it = m_messages.constFind(TranslatorMessage(context, sourceText, QString(), QString(), fileName, lineNumber));
-        if (it != m_messages.constEnd())
-            return it.key();
+    ByteTranslatorMessage bmsg(originalBytes(message.context(), isUtf8),
+                               originalBytes(message.sourceText(), isUtf8),
+                               originalBytes(message.comment(), isUtf8),
+                               message.translations());
+    if (!forceComment) {
+        ByteTranslatorMessage bmsg2(
+                bmsg.context(), bmsg.sourceText(), QByteArray(""), bmsg.translations());
+        if (!m_messages.contains(bmsg2)) {
+            m_messages.insert(bmsg2, 0);
+            return;
+        }
     }
-
-    it = m_messages.constFind(TranslatorMessage(context, QString(), comment, QString(), fileName, lineNumber));
-    if (it != m_messages.constEnd())
-        return it.key();
-    if (comment.isEmpty())
-        return TranslatorMessage();
-
-    it = m_messages.constFind(TranslatorMessage(context, QString(), QString(), QString(), fileName, lineNumber));
-    if (it != m_messages.constEnd())
-        return it.key();
-    return TranslatorMessage();
+    m_messages.insert(bmsg, 0);
 }
 
-bool Releaser::isEmpty() const
+void Releaser::insert(const TranslatorMessage &message, bool forceComment)
 {
-    return m_messageArray.isEmpty() && m_offsetArray.isEmpty()
-        && m_contextArray.isEmpty() && m_messages.isEmpty();
+    insertInternal(message, forceComment, message.isUtf8());
+    if (message.isUtf8() && message.isNonUtf8())
+        insertInternal(message, forceComment, false);
 }
 
 void Releaser::setNumerusRules(const QByteArray &rules)
@@ -463,11 +445,6 @@ void Releaser::setNumerusRules(const QByteArray &rules)
     m_numerusRules = rules;
 }
 
-QList<TranslatorMessage> Releaser::messages() const
-{
-    return m_messages.keys();
-}
-
 static quint8 read8(const uchar *data)
 {
     return *data;
@@ -478,6 +455,31 @@ static quint32 read32(const uchar *data)
     return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | (data[3]);
 }
 
+static void fromBytes(const char *str, int len, QTextCodec *codec, QTextCodec *utf8Codec,
+                      QString *out, QString *utf8Out,
+                      bool *isSystem, bool *isUtf8, bool *needs8Bit)
+{
+    for (int i = 0; i < len; ++i)
+        if (str[i] & 0x80) {
+            if (utf8Codec) {
+                QTextCodec::ConverterState cvtState;
+                *utf8Out = utf8Codec->toUnicode(str, len, &cvtState);
+                *isUtf8 = !cvtState.invalidChars;
+            }
+            QTextCodec::ConverterState cvtState;
+            *out = codec->toUnicode(str, len, &cvtState);
+            *isSystem = !cvtState.invalidChars;
+            *needs8Bit = true;
+            return;
+        }
+    *out = QString::fromLatin1(str, len);
+    *isSystem = true;
+    if (utf8Codec) {
+        *utf8Out = *out;
+        *isUtf8 = true;
+    }
+    *needs8Bit = false;
+}
 
 bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
 {
@@ -543,10 +545,19 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
     size_t numItems = offsetLength / (2 * sizeof(quint32));
     //qDebug() << "NUMITEMS: " << numItems;
 
-    TranslatorMessage msg;
-
     // FIXME: that's just a guess, the original locale data is lost...
     QTextCodec *codec = QTextCodec::codecForLocale();
+    QTextCodec *utf8Codec = 0;
+    if (codec->name() != "UTF-8")
+        utf8Codec = QTextCodec::codecForName("UTF-8");
+
+    QString context, contextUtf8;
+    bool contextIsSystem, contextIsUtf8, contextNeeds8Bit;
+    QString sourcetext, sourcetextUtf8;
+    bool sourcetextIsSystem, sourcetextIsUtf8, sourcetextNeeds8Bit;
+    QString comment, commentUtf8;
+    bool commentIsSystem, commentIsUtf8, commentNeeds8Bit;
+    QStringList translations;
 
     for (const uchar *start = offsetArray; start != offsetArray + (numItems << 3); start += 8) {
         //quint32 hash = read32(start);
@@ -575,7 +586,7 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 }
                 str.replace(QChar(Translator::InternalVariantSeparator),
                             QChar(Translator::DefaultVariantSeparator));
-                msg.appendTranslation(str);
+                translations << str;
                 m += len;
                 break;
             }
@@ -588,7 +599,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 m += 4;
                 //qDebug() << "SOURCE LEN: " << len;
                 //qDebug() << "SOURCE: " << QByteArray((const char*)m, len);
-                msg.setSourceText(codec->toUnicode(QByteArray((const char*)m, len)));
+                fromBytes((const char*)m, len, codec, utf8Codec,
+                          &sourcetext, &sourcetextUtf8,
+                          &sourcetextIsSystem, &sourcetextIsUtf8, &sourcetextNeeds8Bit);
                 m += len;
                 break;
             }
@@ -597,7 +610,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 m += 4;
                 //qDebug() << "CONTEXT LEN: " << len;
                 //qDebug() << "CONTEXT: " << QByteArray((const char*)m, len);
-                msg.setContext(codec->toUnicode(QByteArray((const char*)m, len)));
+                fromBytes((const char*)m, len, codec, utf8Codec,
+                          &context, &contextUtf8,
+                          &contextIsSystem, &contextIsUtf8, &contextNeeds8Bit);
                 m += len;
                 break;
             }
@@ -606,7 +621,9 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
                 m += 4;
                 //qDebug() << "COMMENT LEN: " << len;
                 //qDebug() << "COMMENT: " << QByteArray((const char*)m, len);
-                msg.setComment(codec->toUnicode(QByteArray((const char*)m, len)));
+                fromBytes((const char*)m, len, codec, utf8Codec,
+                          &comment, &commentUtf8,
+                          &commentIsSystem, &commentIsUtf8, &commentNeeds8Bit);
                 m += len;
                 break;
             }
@@ -616,11 +633,33 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
             }
         }
     end:;
+        TranslatorMessage msg;
         msg.setType(TranslatorMessage::Finished);
+        msg.setTranslations(translations);
+        translations.clear();
+        if (contextNeeds8Bit || sourcetextNeeds8Bit || commentNeeds8Bit) {
+            if (utf8Codec && contextIsUtf8 && sourcetextIsUtf8 && commentIsUtf8) {
+                // The message is utf-8, but file is not.
+                msg.setUtf8(true);
+                msg.setContext(contextUtf8);
+                msg.setSourceText(sourcetextUtf8);
+                msg.setComment(commentUtf8);
+                translator.append(msg);
+                continue;
+            }
+            if (!(contextIsSystem && sourcetextIsSystem && commentIsSystem)) {
+                cd.appendError(QLatin1String(
+                        "Cannot read file with current system character codec"));
+                return false;
+            }
+            // The message is 8-bit in the file's encoding (utf-8 or not).
+        }
+        msg.setContext(context);
+        msg.setSourceText(sourcetext);
+        msg.setComment(comment);
         translator.append(msg);
-        //qDebug() << "\nHASH:" << hash << msg.sourceText() << msg.context();
-        msg.setTranslations(QStringList());
     }
+    translator.resolveDualEncoded();
     return ok;
 }
 
@@ -657,30 +696,16 @@ static bool saveQM(const Translator &translator, QIODevice &dev, ConversionData
             } else {
                 ++finished;
             }
-            QString context = msg.context();
-            QString sourceText = msg.sourceText();
-            QString comment = msg.comment();
-            QStringList translations = msg.translations();
-
-                /*
-                Drop the comment in (context, sourceText, comment),
-                unless the context is empty,
-                unless (context, sourceText, "") already exists or
-                unless we already dropped the comment of (context,
-                sourceText, comment0).
-                */
-                if (comment.isEmpty()
-                    || context.isEmpty()
-                    || translator.contains(context, sourceText, QString())
-                    || !releaser.findMessage(context, sourceText, QString()).translation()
-                            .isNull() ) {
-                    releaser.insert(msg);
-                } else {
-                    TranslatorMessage tm(context, sourceText, QString(),
-                    QString(), QString(), -1, translations);
-                    //filename and lineNumbers will be ignored from now.
-                    releaser.insert(tm);
-                }
+            // Drop the comment in (context, sourceText, comment),
+            // unless the context is empty,
+            // unless (context, sourceText, "") already exists or
+            // unless we already dropped the comment of (context,
+            // sourceText, comment0).
+            bool forceComment =
+                    msg.comment().isEmpty()
+                    || msg.context().isEmpty()
+                    || translator.contains(msg.context(), msg.sourceText(), QString());
+            releaser.insert(msg, forceComment);
         }
     }
 
diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp
index 32404a5f7a..3721204173 100644
--- a/tools/linguist/shared/translator.cpp
+++ b/tools/linguist/shared/translator.cpp
@@ -114,6 +114,10 @@ void Translator::extend(const TranslatorMessage &msg)
             cmt.append(msg.extraComment());
             emsg.setExtraComment(cmt);
         }
+        if (msg.isUtf8() != emsg.isUtf8()) {
+            emsg.setUtf8(true);
+            emsg.setNonUtf8(true);
+        }
     }
 }
 
@@ -425,6 +429,28 @@ QList<TranslatorMessage> Translator::findDuplicates() const
     return ret;
 }
 
+void Translator::resolveDualEncoded()
+{
+    QHash<TranslatorMessage, int> dups;
+    for (int i = 0; i < m_messages.count();) {
+        const TranslatorMessage &msg = m_messages.at(i);
+        QHash<TranslatorMessage, int>::ConstIterator it = dups.constFind(msg);
+        if (it != dups.constEnd()) {
+            TranslatorMessage &omsg = m_messages[*it];
+            if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {
+                omsg.setUtf8(true);
+                omsg.setNonUtf8(true);
+                m_messages.removeAt(i);
+                continue;
+            }
+            // Regular dupe; will complain later
+        } else {
+            dups[msg] = i;
+        }
+        ++i;
+    }
+}
+
 // Used by lupdate to be able to search using absolute paths during merging
 void Translator::makeFileNamesAbsolute(const QDir &originalPath)
 {
diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h
index bbe765443e..ba719ec56c 100644
--- a/tools/linguist/shared/translator.h
+++ b/tools/linguist/shared/translator.h
@@ -130,6 +130,7 @@ public:
     void stripIdenticalSourceTranslations();
     void dropTranslations();
     QList<TranslatorMessage> findDuplicates() const;
+    void resolveDualEncoded();
     void makeFileNamesAbsolute(const QDir &originalPath);
 
     void setCodecName(const QByteArray &name);
diff --git a/tools/linguist/shared/translatormessage.cpp b/tools/linguist/shared/translatormessage.cpp
index ab4301fbb3..afe66fe888 100644
--- a/tools/linguist/shared/translatormessage.cpp
+++ b/tools/linguist/shared/translatormessage.cpp
@@ -54,7 +54,7 @@
 QT_BEGIN_NAMESPACE
 
 TranslatorMessage::TranslatorMessage()
-  : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_plural(false)
+  : m_lineNumber(-1), m_type(Unfinished), m_utf8(false), m_nonUtf8(false), m_plural(false)
 {
 }
 
@@ -66,7 +66,7 @@ TranslatorMessage::TranslatorMessage(const QString &context,
   : m_context(context), m_sourcetext(sourceText), m_comment(comment),
     m_userData(userData),
     m_translations(translations), m_fileName(fileName), m_lineNumber(lineNumber),
-    m_type(type), m_utf8(false), m_plural(plural)
+    m_type(type), m_utf8(false), m_nonUtf8(false), m_plural(plural)
 {
 }
 
diff --git a/tools/linguist/shared/translatormessage.h b/tools/linguist/shared/translatormessage.h
index fa37ff5ce7..2818e28686 100644
--- a/tools/linguist/shared/translatormessage.h
+++ b/tools/linguist/shared/translatormessage.h
@@ -136,6 +136,8 @@ public:
     void setType(Type t) { m_type = t; }
     bool isUtf8() const { return m_utf8; } // codecForTr override
     void setUtf8(bool on) { m_utf8 = on; }
+    bool isNonUtf8() const { return m_nonUtf8; } // codecForTr override
+    void setNonUtf8(bool on) { m_nonUtf8 = on; }
     bool isPlural() const { return m_plural; }
     void setPlural(bool isplural) { m_plural = isplural; }
 
@@ -169,6 +171,7 @@ private:
 
     Type m_type;
     bool m_utf8;
+    bool m_nonUtf8;
     bool m_plural;
 };
 
diff --git a/tools/linguist/shared/ts.cpp b/tools/linguist/shared/ts.cpp
index 3af9e593c6..8a6d365a8e 100644
--- a/tools/linguist/shared/ts.cpp
+++ b/tools/linguist/shared/ts.cpp
@@ -212,6 +212,7 @@ QString TSReader::readTransContents()
 
 bool TSReader::read(Translator &translator)
 {
+    STRING(both);
     STRING(byte);
     STRING(comment);
     STRING(context);
@@ -265,13 +266,15 @@ bool TSReader::read(Translator &translator)
             QString currentFile;
 
             QXmlStreamAttributes atts = attributes();
-            //QString version = atts.value(strversion).toString();
+            QString version = atts.value(strversion).toString();
             translator.setLanguageCode(atts.value(strlanguage).toString());
             translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());
             while (!atEnd()) {
                 readNext();
                 if (isEndElement()) {
                     // </TS> found, finish local loop
+                    if (version == QLatin1String("1.1"))
+                        translator.resolveDualEncoded();
                     break;
                 } else if (isWhiteSpace()) {
                     // ignore these, just whitespace
@@ -311,7 +314,9 @@ bool TSReader::read(Translator &translator)
                             msg.setContext(context);
                             msg.setType(TranslatorMessage::Finished);
                             msg.setPlural(attributes().value(strnumerus) == stryes);
-                            msg.setUtf8(attributes().value(strutf8) == strtrue
+                            const QStringRef &utf8Attr = attributes().value(strutf8);
+                            msg.setNonUtf8(utf8Attr == strboth);
+                            msg.setUtf8(msg.isNonUtf8() || utf8Attr == strtrue
                                  ||  attributes().value(strencoding) == strUtf8);
                             while (!atEnd()) {
                                 readNext();
@@ -594,107 +599,126 @@ bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd, in
         foreach (const TranslatorMessage &msg, messageOrder[context]) {
             //msg.dump();
 
-            t << "    <message";
-            if (!msg.id().isEmpty())
-                t << " id=\"" << msg.id() << "\"";
-            if (format == 11 && !trIsUtf8 && msg.isUtf8())
-                t << " encoding=\"UTF-8\"";
-            if (format == 20 && !trIsUtf8 && msg.isUtf8()) 
-                t << " utf8=\"true\"";
-            if (msg.isPlural())
-                t << " numerus=\"yes\"";
-            t << ">\n";
-            if (translator.locationsType() != Translator::NoLocations) {
-                QString cfile = currentFile;
-                bool first = true;
-                foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
-                    QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
-                                .replace(QLatin1Char('\\'),QLatin1Char('/'));
-                    int ln = ref.lineNumber();
-                    QString ld;
-                    if (translator.locationsType() == Translator::RelativeLocations) {
-                        if (ln != -1) {
-                            int dlt = ln - currentLine[fn];
-                            if (dlt >= 0)
-                                ld.append(QLatin1Char('+'));
-                            ld.append(QString::number(dlt));
-                            currentLine[fn] = ln;
+            bool isUtf8 = msg.isUtf8();
+            bool second = false;
+            forever {
+
+                t << "    <message";
+                if (!msg.id().isEmpty())
+                    t << " id=\"" << msg.id() << "\"";
+                if (!trIsUtf8) {
+                    if (format == 11) {
+                        if (isUtf8)
+                            t << " encoding=\"UTF-8\"";
+                    } else {
+                        if (msg.isUtf8()) {
+                            if (msg.isNonUtf8())
+                                t << " utf8=\"both\"";
+                            else
+                                t << " utf8=\"true\"";
                         }
+                    }
+                }
+                if (msg.isPlural())
+                    t << " numerus=\"yes\"";
+                t << ">\n";
+                if (translator.locationsType() != Translator::NoLocations) {
+                    QString cfile = currentFile;
+                    bool first = true;
+                    foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
+                        QString fn = cd.m_targetDir.relativeFilePath(ref.fileName())
+                                    .replace(QLatin1Char('\\'),QLatin1Char('/'));
+                        int ln = ref.lineNumber();
+                        QString ld;
+                        if (translator.locationsType() == Translator::RelativeLocations) {
+                            if (ln != -1) {
+                                int dlt = ln - currentLine[fn];
+                                if (dlt >= 0)
+                                    ld.append(QLatin1Char('+'));
+                                ld.append(QString::number(dlt));
+                                currentLine[fn] = ln;
+                            }
 
-                        if (fn != cfile) {
-                            if (first)
-                                currentFile = fn;
-                            cfile = fn;
+                            if (fn != cfile) {
+                                if (first)
+                                    currentFile = fn;
+                                cfile = fn;
+                            } else {
+                                fn.clear();
+                            }
+                            first = false;
                         } else {
-                            fn.clear();
+                            if (ln != -1)
+                                ld = QString::number(ln);
                         }
-                        first = false;
-                    } else {
-                        if (ln != -1)
-                            ld = QString::number(ln);
+                        t << "        <location";
+                        if (!fn.isEmpty())
+                            t << " filename=\"" << fn << "\"";
+                        if (!ld.isEmpty())
+                            t << " line=\"" << ld << "\"";
+                        t << "/>\n";
                     }
-                    t << "        <location";
-                    if (!fn.isEmpty())
-                        t << " filename=\"" << fn << "\"";
-                    if (!ld.isEmpty())
-                        t << " line=\"" << ld << "\"";
-                    t << "/>\n";
                 }
-            }
 
-            t << "        <source>"
-              << evilBytes(msg.sourceText(), msg.isUtf8(), format, codecName)
-              << "</source>\n";
+                t << "        <source>"
+                  << evilBytes(msg.sourceText(), isUtf8, format, codecName)
+                  << "</source>\n";
 
-            if (format != 11 && !msg.oldSourceText().isEmpty())
-                t << "        <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
+                if (format != 11 && !msg.oldSourceText().isEmpty())
+                    t << "        <oldsource>" << protect(msg.oldSourceText()) << "</oldsource>\n";
 
-            if (!msg.comment().isEmpty()) {
-                t << "        <comment>"
-                  << evilBytes(msg.comment(), msg.isUtf8(), format, codecName)
-                  << "</comment>\n";
-            }
+                if (!msg.comment().isEmpty()) {
+                    t << "        <comment>"
+                      << evilBytes(msg.comment(), isUtf8, format, codecName)
+                      << "</comment>\n";
+                }
 
-            if (format != 11) {
+                if (format != 11) {
 
-                if (!msg.oldComment().isEmpty())
-                    t << "        <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
+                    if (!msg.oldComment().isEmpty())
+                        t << "        <oldcomment>" << protect(msg.oldComment()) << "</oldcomment>\n";
 
-                if (!msg.extraComment().isEmpty())
-                    t << "        <extracomment>" << protect(msg.extraComment())
-                      << "</extracomment>\n";
+                    if (!msg.extraComment().isEmpty())
+                        t << "        <extracomment>" << protect(msg.extraComment())
+                          << "</extracomment>\n";
 
-                if (!msg.translatorComment().isEmpty())
-                    t << "        <translatorcomment>" << protect(msg.translatorComment())
-                      << "</translatorcomment>\n";
+                    if (!msg.translatorComment().isEmpty())
+                        t << "        <translatorcomment>" << protect(msg.translatorComment())
+                          << "</translatorcomment>\n";
 
-            }
+                }
 
-            t << "        <translation";
-            if (msg.type() == TranslatorMessage::Unfinished)
-                t << " type=\"unfinished\"";
-            else if (msg.type() == TranslatorMessage::Obsolete)
-                t << " type=\"obsolete\"";
-            if (msg.isPlural()) {
-                t << ">";
-                QStringList translns = translator.normalizedTranslations(msg, cd, &result);
-                for (int j = 0; j < qMax(1, translns.count()); ++j) {
-                    t << "\n            <numerusform";
-                    writeVariants(t, "            ", translns[j]);
-                    t << "</numerusform>";
+                t << "        <translation";
+                if (msg.type() == TranslatorMessage::Unfinished)
+                    t << " type=\"unfinished\"";
+                else if (msg.type() == TranslatorMessage::Obsolete)
+                    t << " type=\"obsolete\"";
+                if (msg.isPlural()) {
+                    t << ">";
+                    QStringList translns = translator.normalizedTranslations(msg, cd, &result);
+                    for (int j = 0; j < qMax(1, translns.count()); ++j) {
+                        t << "\n            <numerusform";
+                        writeVariants(t, "            ", translns[j]);
+                        t << "</numerusform>";
+                    }
+                    t << "\n        ";
+                } else {
+                    writeVariants(t, "        ", msg.translation());
                 }
-                t << "\n        ";
-            } else {
-                writeVariants(t, "        ", msg.translation());
-            }
-            t << "</translation>\n";
+                t << "</translation>\n";
+
+                if (format != 11)
+                    writeExtras(t, "        ", msg.extras(), drops);
 
-            if (format != 11)
-                writeExtras(t, "        ", msg.extras(), drops);
+                if (!msg.userData().isEmpty())
+                    t << "        <userdata>" << msg.userData() << "</userdata>\n";
+                t << "    </message>\n";
 
-            if (!msg.userData().isEmpty())
-                t << "        <userdata>" << msg.userData() << "</userdata>\n";
-            t << "    </message>\n";
+                if (format != 11 || second || !msg.isUtf8() || !msg.isNonUtf8())
+                    break;
+                isUtf8 = false;
+                second = true;
+            }
         }
         t << "</context>\n";
     }