handle messages with ids (more) correctly

when comparing two messages which both have ids, compare only the ids.
this affects finding/reporting duplicates and merging.

13 files changed, 239 insertions, 53 deletions

diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp/finddialog.cpp b/tests/auto/linguist/lupdate/testdata/good/mergecpp/finddialog.cpp
index 53eba32fb6..f27e1f036c 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp/finddialog.cpp
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp/finddialog.cpp
@@ -63,5 +63,16 @@ void FindDialog::reset()
 {
     tr("%n item(s)", "merge from singular to plural form", 4);
     tr("%n item(s)", "merge from a finished singular form to an unfinished plural form", 4);
-}
 
+
+
+    //% "Hello"
+    qtTrId("xx_hello");
+
+    //% "New world"
+    qtTrId("xx_world");
+
+
+    //= new_id
+    tr("this is just some text");
+}
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.before b/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.before
index d06252cec2..ad2f65fd49 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.before
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.before
@@ -1,6 +1,19 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE TS><TS version="1.1" language="zh_CN">
 <context>
+    <name></name>
+    <message id="xx_hello">
+        <location filename="finddialog.cpp" line="70"/>
+        <source>Hello</source>
+        <translation>Hallo</translation>
+    </message>
+    <message id="xx_world">
+        <location filename="finddialog.cpp" line="73"/>
+        <source>World</source>
+        <translation>Welt</translation>
+    </message>
+</context>
+<context>
     <name>FindDialog</name>
     <message>
         <source></source>
@@ -44,5 +57,10 @@
             <numerusform></numerusform>
         </translation>
     </message>
+    <message>
+        <location filename="finddialog.cpp" line="59"/>
+        <source>this is just some text</source>
+        <translation type="unfinished">Unfertige Uebersetzung</translation>
+    </message>
 </context>
 </TS>
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.result b/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.result
index be4e02ee9d..e9abf89587 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.result
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp/project.ts.result
@@ -2,6 +2,20 @@
 <!DOCTYPE TS>
 <TS version="2.0" language="zh_CN">
 <context>
+    <name></name>
+    <message id="xx_hello">
+        <location filename="finddialog.cpp" line="70"/>
+        <source>Hello</source>
+        <translation>Hallo</translation>
+    </message>
+    <message id="xx_world">
+        <location filename="finddialog.cpp" line="73"/>
+        <source>New world</source>
+        <oldsource>World</oldsource>
+        <translation type="unfinished">Welt</translation>
+    </message>
+</context>
+<context>
     <name>FindDialog</name>
     <message>
         <source></source>
@@ -45,5 +59,10 @@
             <numerusform></numerusform>
         </translation>
     </message>
+    <message id="new_id">
+        <location filename="finddialog.cpp" line="77"/>
+        <source>this is just some text</source>
+        <translation type="unfinished">Unfertige Uebersetzung</translation>
+    </message>
 </context>
 </TS>
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/finddialog.cpp b/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/finddialog.cpp
index e1464a2d65..9abb367ec7 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/finddialog.cpp
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/finddialog.cpp
@@ -148,4 +148,7 @@ void FindDialog::doFind(bool forward)
 bool FindDialog::hasFindExpression() const
 {
     // statusMessage(tr( "Should be obsolete" ));
+
+    //% "This is some random text"
+    qtTrId("keep_id")
 }
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.before b/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.before
index 834f5121eb..feab1694d9 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.before
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.before
@@ -1,6 +1,19 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE TS><TS version="1.1">
 <context>
+    <name></name>
+    <message id="keep_id">
+        <location filename="finddialog.cpp" line="153"/>
+        <source>This is some random text</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message id="obsolete_id">
+        <location filename="finddialog.cpp" line="155"/>
+        <source>Should be obsolete, too</source>
+        <translation type="unfinished">SHOULD BE OBSOLETE AS WELL</translation>
+    </message>
+</context>
+<context>
     <name>FindDialog</name>
     <message>
         <location filename="finddialog.cpp" line="85"/>
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.result b/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.result
index b328e90482..ee3d0f69bb 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.result
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp_noobsolete/project.ts.result
@@ -2,6 +2,14 @@
 <!DOCTYPE TS>
 <TS version="2.0">
 <context>
+    <name></name>
+    <message id="keep_id">
+        <location filename="finddialog.cpp" line="153"/>
+        <source>This is some random text</source>
+        <translation type="unfinished"></translation>
+    </message>
+</context>
+<context>
     <name>FindDialog</name>
     <message>
         <location filename="finddialog.cpp" line="85"/>
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/finddialog.cpp b/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/finddialog.cpp
index 7b28c75e19..cc3af488e9 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/finddialog.cpp
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/finddialog.cpp
@@ -150,6 +150,9 @@ void FindDialog::doFind(bool forward)
 
 bool FindDialog::hasFindExpression() const
 {
+    //% "This is some random text"
+    qtTrId("keep_id")
+
     return !findExpr.isEmpty();
 }
 
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.before b/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.before
index 1fa0fd35fc..2bc60491c4 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.before
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.before
@@ -1,6 +1,19 @@
 <?xml version="1.0"?>
 <!DOCTYPE TS><TS version="1.1">
 <context>
+    <name></name>
+    <message id="keep_id">
+        <location filename="finddialog.cpp" line="154"/>
+        <source>This is some random text</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message id="obsolete_id">
+        <location filename="finddialog.cpp" line="155"/>
+        <source>Should be obsolete, too</source>
+        <translation type="unfinished">SHOULD BE OBSOLETE AS WELL</translation>
+    </message>
+</context>
+<context>
     <name>FindDialog</name>
     <message>
         <location filename="finddialog.cpp" line="85"/>
diff --git a/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.result b/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.result
index cfd11b1806..f442cbcc99 100644
--- a/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.result
+++ b/tests/auto/linguist/lupdate/testdata/good/mergecpp_obsolete/project.ts.result
@@ -2,6 +2,18 @@
 <!DOCTYPE TS>
 <TS version="2.0">
 <context>
+    <name></name>
+    <message id="keep_id">
+        <location filename="finddialog.cpp" line="154"/>
+        <source>This is some random text</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message id="obsolete_id">
+        <source>Should be obsolete, too</source>
+        <translation type="obsolete">SHOULD BE OBSOLETE AS WELL</translation>
+    </message>
+</context>
+<context>
     <name>FindDialog</name>
     <message>
         <source>Enter the text you are looking for.</source>
diff --git a/tools/linguist/linguist/messagemodel.cpp b/tools/linguist/linguist/messagemodel.cpp
index 89666242e9..024fd9110a 100644
--- a/tools/linguist/linguist/messagemodel.cpp
+++ b/tools/linguist/linguist/messagemodel.cpp
@@ -210,9 +210,16 @@ bool DataModel::load(const QString &fileName, bool *langGuessed, QWidget *parent
     }
 
     Translator::Duplicates dupes = tor.resolveDuplicates();
-    if (!dupes.byContents.isEmpty()) {
+    if (!dupes.byId.isEmpty() || !dupes.byContents.isEmpty()) {
         QString err = tr("<qt>Duplicate messages found in '%1':").arg(Qt::escape(fileName));
         int numdups = 0;
+        foreach (int i, dupes.byId) {
+            if (++numdups >= 5) {
+                err += tr("<p>[more duplicates omitted]");
+                goto doWarn;
+            }
+            err += tr("<p>* ID: %1").arg(Qt::escape(tor.message(i).id()));
+        }
         foreach (int j, dupes.byContents) {
             const TranslatorMessage &msg = tor.message(j);
             if (++numdups >= 5) {
@@ -224,6 +231,7 @@ bool DataModel::load(const QString &fileName, bool *langGuessed, QWidget *parent
             if (!msg.comment().isEmpty())
                 err += tr("<br>* Comment: %3").arg(Qt::escape(msg.comment()));
         }
+      doWarn:
         QMessageBox::warning(parent, QObject::tr("Qt Linguist"), err);
     }
 
diff --git a/tools/linguist/lupdate/merge.cpp b/tools/linguist/lupdate/merge.cpp
index 13ba4ae4e1..a72143939a 100644
--- a/tools/linguist/lupdate/merge.cpp
+++ b/tools/linguist/lupdate/merge.cpp
@@ -375,6 +375,7 @@ Translator merge(const Translator &tor, const Translator &virginTor,
                             ++similarTextHeuristicCount;
                             neww++;
 
+                          outdateSource:
                             m.setOldSourceText(m.sourceText());
                             m.setSourceText(mv.sourceText());
                             const QString &oldpluralsource = m.extra(QLatin1String("po-msgid_plural"));
@@ -392,23 +393,35 @@ Translator merge(const Translator &tor, const Translator &virginTor,
                 }
             } else {
                 mv = virginTor.message(mvi);
-                switch (m.type()) {
-                case TranslatorMessage::Finished:
-                default:
-                    if (m.isPlural() == mv.isPlural()) {
-                        newType = TranslatorMessage::Finished;
-                    } else {
-                        newType = TranslatorMessage::Unfinished;
-                    }
+                if (!mv.id().isEmpty()
+                    && (mv.context() != m.context()
+                        || mv.sourceText() != m.sourceText()
+                        || mv.comment() != m.comment())) {
                     known++;
-                    break;
-                case TranslatorMessage::Unfinished:
                     newType = TranslatorMessage::Unfinished;
-                    known++;
-                    break;
-                case TranslatorMessage::Obsolete:
-                    newType = TranslatorMessage::Unfinished;
-                    neww++;
+                    m.setContext(mv.context());
+                    m.setComment(mv.comment());
+                    if (mv.sourceText() != m.sourceText())
+                        goto outdateSource;
+                } else {
+                    switch (m.type()) {
+                    case TranslatorMessage::Finished:
+                    default:
+                        if (m.isPlural() == mv.isPlural()) {
+                            newType = TranslatorMessage::Finished;
+                        } else {
+                            newType = TranslatorMessage::Unfinished;
+                        }
+                        known++;
+                        break;
+                    case TranslatorMessage::Unfinished:
+                        newType = TranslatorMessage::Unfinished;
+                        known++;
+                        break;
+                    case TranslatorMessage::Obsolete:
+                        newType = TranslatorMessage::Unfinished;
+                        neww++;
+                    }
                 }
 
                 // Always get the filename and linenumber info from the
@@ -421,6 +434,7 @@ Translator merge(const Translator &tor, const Translator &virginTor,
                 m.setPlural(mv.isPlural());
                 m.setUtf8(mv.isUtf8());
                 m.setExtraComment(mv.extraComment());
+                m.setId(mv.id());
             }
         }
 
diff --git a/tools/linguist/shared/translator.cpp b/tools/linguist/shared/translator.cpp
index 0a59668974..77faabda51 100644
--- a/tools/linguist/shared/translator.cpp
+++ b/tools/linguist/shared/translator.cpp
@@ -306,10 +306,15 @@ int Translator::find(const TranslatorMessage &msg) const
 {
     for (int i = 0; i < m_messages.count(); ++i) {
         const TranslatorMessage &tmsg = m_messages.at(i);
-        if (msg.context() == tmsg.context()
-            && msg.sourceText() == tmsg.sourceText()
-            && msg.comment() == tmsg.comment())
-            return i;
+        if (msg.id().isEmpty() || tmsg.id().isEmpty()) {
+            if (msg.context() == tmsg.context()
+                && msg.sourceText() == tmsg.sourceText()
+                && msg.comment() == tmsg.comment())
+                return i;
+        } else {
+            if (msg.id() == tmsg.id())
+                return i;
+        }
     }
     return -1;
 }
@@ -423,8 +428,34 @@ void Translator::dropUiLines()
     }
 }
 
-struct TranslatorMessagePtr {
-    TranslatorMessagePtr(const TranslatorMessage &tm)
+struct TranslatorMessageIdPtr {
+    explicit TranslatorMessageIdPtr(const TranslatorMessage &tm)
+    {
+        ptr = &tm;
+    }
+
+    inline const TranslatorMessage *operator->() const
+    {
+        return ptr;
+    }
+
+    const TranslatorMessage *ptr;
+};
+
+Q_DECLARE_TYPEINFO(TranslatorMessageIdPtr, Q_MOVABLE_TYPE);
+
+inline int qHash(TranslatorMessageIdPtr tmp)
+{
+    return qHash(tmp->id());
+}
+
+inline bool operator==(TranslatorMessageIdPtr tmp1, TranslatorMessageIdPtr tmp2)
+{
+    return tmp1->id() == tmp2->id();
+}
+
+struct TranslatorMessageContentPtr {
+    explicit TranslatorMessageContentPtr(const TranslatorMessage &tm)
     {
         ptr = &tm;
     }
@@ -437,50 +468,81 @@ struct TranslatorMessagePtr {
     const TranslatorMessage *ptr;
 };
 
-Q_DECLARE_TYPEINFO(TranslatorMessagePtr, Q_MOVABLE_TYPE);
+Q_DECLARE_TYPEINFO(TranslatorMessageContentPtr, Q_MOVABLE_TYPE);
 
-inline int qHash(TranslatorMessagePtr tmp)
+inline int qHash(TranslatorMessageContentPtr tmp)
 {
-    return
-        qHash(tmp->context()) ^
-        qHash(tmp->sourceText()) ^
-        qHash(tmp->comment()) ^
-        qHash(tmp->id());
+    int hash = qHash(tmp->context()) ^ qHash(tmp->sourceText());
+    if (!tmp->sourceText().isEmpty())
+        // Special treatment for context comments (empty source).
+        hash ^= qHash(tmp->comment());
+    return hash;
 }
 
-inline bool operator==(TranslatorMessagePtr tmp1, TranslatorMessagePtr tmp2)
+inline bool operator==(TranslatorMessageContentPtr tmp1, TranslatorMessageContentPtr tmp2)
 {
+    if (tmp1->context() != tmp2->context() || tmp1->sourceText() != tmp2->sourceText())
+        return false;
     // Special treatment for context comments (empty source).
-    return (tmp1->context() == tmp2->context())
-        && tmp1->sourceText() == tmp2->sourceText()
-        && tmp1->id() == tmp2->id()
-        && (tmp1->sourceText().isEmpty() || tmp1->comment() == tmp2->comment());
+    if (tmp1->sourceText().isEmpty())
+        return true;
+    return tmp1->comment() == tmp2->comment();
 }
 
 Translator::Duplicates Translator::resolveDuplicates()
 {
     Duplicates dups;
-    QHash<TranslatorMessagePtr, int> refs;
+    QHash<TranslatorMessageIdPtr, int> idRefs;
+    QHash<TranslatorMessageContentPtr, int> contentRefs;
     for (int i = 0; i < m_messages.count();) {
         const TranslatorMessage &msg = m_messages.at(i);
-        QHash<TranslatorMessagePtr, int>::ConstIterator it = refs.constFind(msg);
-        if (it != refs.constEnd()) {
-            TranslatorMessage &omsg = m_messages[*it];
-            if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {
-                // Dual-encoded message
-                omsg.setUtf8(true);
-                omsg.setNonUtf8(true);
-            } else {
-                // Duplicate
-                dups.byContents.insert(*it);
+        TranslatorMessage *omsg;
+        int oi;
+        QSet<int> *pDup;
+        if (!msg.id().isEmpty()) {
+            QHash<TranslatorMessageIdPtr, int>::ConstIterator it =
+                    idRefs.constFind(TranslatorMessageIdPtr(msg));
+            if (it != idRefs.constEnd()) {
+                oi = *it;
+                omsg = &m_messages[oi];
+                pDup = &dups.byId;
+                goto gotDupe;
+            }
+        }
+        {
+            QHash<TranslatorMessageContentPtr, int>::ConstIterator it =
+                    contentRefs.constFind(TranslatorMessageContentPtr(msg));
+            if (it != contentRefs.constEnd()) {
+                oi = *it;
+                omsg = &m_messages[oi];
+                if (msg.id().isEmpty() || omsg->id().isEmpty()) {
+                    if (!msg.id().isEmpty() && omsg->id().isEmpty()) {
+                        omsg->setId(msg.id());
+                        idRefs[TranslatorMessageIdPtr(*omsg)] = oi;
+                    }
+                    pDup = &dups.byContents;
+                    goto gotDupe;
+                }
+                // This is really a content dupe, but with two distinct IDs.
             }
-            if (!omsg.isTranslated() && msg.isTranslated())
-                omsg.setTranslations(msg.translations());
-            m_messages.removeAt(i);
+        }
+        if (!msg.id().isEmpty())
+            idRefs[TranslatorMessageIdPtr(msg)] = i;
+        contentRefs[TranslatorMessageContentPtr(msg)] = i;
+        ++i;
+        continue;
+      gotDupe:
+        if (omsg->isUtf8() != msg.isUtf8() && !omsg->isNonUtf8()) {
+            // Dual-encoded message
+            omsg->setUtf8(true);
+            omsg->setNonUtf8(true);
         } else {
-            refs[msg] = i;
-            ++i;
+            // Duplicate
+            pDup->insert(oi);
         }
+        if (!omsg->isTranslated() && msg.isTranslated())
+            omsg->setTranslations(msg.translations());
+        m_messages.removeAt(i);
     }
     return dups;
 }
@@ -488,12 +550,14 @@ Translator::Duplicates Translator::resolveDuplicates()
 void Translator::reportDuplicates(const Duplicates &dupes,
                                   const QString &fileName, bool verbose)
 {
-    if (!dupes.byContents.isEmpty()) {
+    if (!dupes.byId.isEmpty() || !dupes.byContents.isEmpty()) {
         if (!verbose) {
             qWarning("Warning: dropping duplicate messages in '%s'\n(try -verbose for more info).",
                      qPrintable(fileName));
         } else {
             qWarning("Warning: dropping duplicate messages in '%s':", qPrintable(fileName));
+            foreach (int i, dupes.byId)
+                qWarning("\n* ID: %s", qPrintable(message(i).id()));
             foreach (int j, dupes.byContents) {
                 const TranslatorMessage &msg = message(j);
                 qWarning("\n* Context: %s\n* Source: %s",
diff --git a/tools/linguist/shared/translator.h b/tools/linguist/shared/translator.h
index b957fa28a4..eec704a218 100644
--- a/tools/linguist/shared/translator.h
+++ b/tools/linguist/shared/translator.h
@@ -134,7 +134,7 @@ public:
     void dropUiLines();
     void makeFileNamesAbsolute(const QDir &originalPath);
 
-    struct Duplicates { QSet<int> byContents; };
+    struct Duplicates { QSet<int> byId, byContents; };
     Duplicates resolveDuplicates();
     void reportDuplicates(const Duplicates &dupes, const QString &fileName, bool verbose);