qdoc: Teach qdoc to resolve namespaces

It turns out this bug was caused by modularization,
which created the situation where members of the Qt
namespace are in different modules. Most are in QtCore,
but a few are in QtGui.

qdoc was creating a namespace node for the Qt namespace
in the node tree for QtCore, and another namespace node
in the node tree for QtGui. This meant that there were
two NamespaceNodes for the Qt namespace. Correctly, only
one of these nodes contained the text for the \namespace
command for the Qt namespace. This was the namespace node
that was being used to create the HTML reference page for
the Qt namespace.

Unfortunately, the Qt namespace node in the tree for QtGui
was not being merged into the Qt namespace node in QtCore,
so some of the members of the Qt namespace were not being
shown on the reference page.

This update teches qdoc how to merge namespace nodes to
ensure that all the members appear on the reference page
for the namespace. There can be a namespace node for the
namespace xxx in any number of modules, but they will all
be merged into the namespace node for namespace xxx that
contains the qdoc comment for \namespace xxx.

Change-Id: I0f6a653ea6f920aacd5d8e13f9865488d95f6458
Task-number: QTBUG-44688
Reviewed-by: Topi Reiniö <topi.reinio@digia.com>

8 files changed, 153 insertions, 89 deletions

diff --git a/src/tools/qdoc/codemarker.cpp b/src/tools/qdoc/codemarker.cpp
index 6a1675aeb5..2253aca183 100644
--- a/src/tools/qdoc/codemarker.cpp
+++ b/src/tools/qdoc/codemarker.cpp
@@ -391,12 +391,11 @@ void CodeMarker::insert(FastSection &fastSection,
     bool inheritedMember = false;
     if (!node->relates()) {
         InnerNode* p = node->parent();
-        if (p->type() == Node::QmlPropertyGroup)
+        if (p->isQmlPropertyGroup())
             p = p->parent();
         if (p != fastSection.parent_) {
-            if ((!p->isQmlType() && !p->isJsType()) || !p->isAbstract()) {
+            if ((!p->isQmlType() && !p->isJsType()) || !p->isAbstract())
                 inheritedMember = true;
-            }
         }
     }
 
@@ -438,7 +437,7 @@ void CodeMarker::insert(FastSection &fastSection,
             fastSection.memberMap.insertMulti(key, node);
         }
         else {
-            if (node->parent()->type() == Node::Class) {
+            if (node->parent()->isClass() || node->parent()->isNamespace()) {
                 if (fastSection.inherited.isEmpty()
                         || fastSection.inherited.last().first != node->parent()) {
                     QPair<InnerNode *, int> p(node->parent(), 0);
diff --git a/src/tools/qdoc/cppcodemarker.cpp b/src/tools/qdoc/cppcodemarker.cpp
index 400c4808ed..1546b5226e 100644
--- a/src/tools/qdoc/cppcodemarker.cpp
+++ b/src/tools/qdoc/cppcodemarker.cpp
@@ -442,75 +442,73 @@ QList<Section> CppCodeMarker::sections(const InnerNode *inner,
 {
     QList<Section> sections;
 
-    if (inner->type() == Node::Class) {
-        const ClassNode *classNode = static_cast<const ClassNode *>(inner);
-
+    if (inner->isClass()) {
         if (style == Summary) {
-            FastSection privateFunctions(classNode,
+            FastSection privateFunctions(inner,
                                          "Private Functions",
                                          QString(),
                                          "private function",
                                          "private functions");
-            FastSection privateSlots(classNode, "Private Slots", QString(), "private slot", "private slots");
-            FastSection privateTypes(classNode, "Private Types", QString(), "private type", "private types");
-            FastSection protectedFunctions(classNode,
+            FastSection privateSlots(inner, "Private Slots", QString(), "private slot", "private slots");
+            FastSection privateTypes(inner, "Private Types", QString(), "private type", "private types");
+            FastSection protectedFunctions(inner,
                                            "Protected Functions",
                                            QString(),
                                            "protected function",
                                            "protected functions");
-            FastSection protectedSlots(classNode,
+            FastSection protectedSlots(inner,
                                        "Protected Slots",
                                        QString(),
                                        "protected slot",
                                        "protected slots");
-            FastSection protectedTypes(classNode,
+            FastSection protectedTypes(inner,
                                        "Protected Types",
                                        QString(),
                                        "protected type",
                                        "protected types");
-            FastSection protectedVariables(classNode,
+            FastSection protectedVariables(inner,
                                            "Protected Variables",
                                            QString(),
                                            "protected type",
                                            "protected variables");
-            FastSection publicFunctions(classNode,
+            FastSection publicFunctions(inner,
                                         "Public Functions",
                                         QString(),
                                         "public function",
                                         "public functions");
-            FastSection publicSignals(classNode, "Signals", QString(), "signal", "signals");
-            FastSection publicSlots(classNode, "Public Slots", QString(), "public slot", "public slots");
-            FastSection publicTypes(classNode, "Public Types", QString(), "public type", "public types");
-            FastSection publicVariables(classNode,
+            FastSection publicSignals(inner, "Signals", QString(), "signal", "signals");
+            FastSection publicSlots(inner, "Public Slots", QString(), "public slot", "public slots");
+            FastSection publicTypes(inner, "Public Types", QString(), "public type", "public types");
+            FastSection publicVariables(inner,
                                         "Public Variables",
                                         QString(),
                                         "public variable",
                                         "public variables");
-            FastSection properties(classNode, "Properties", QString(), "property", "properties");
-            FastSection relatedNonMembers(classNode,
+            FastSection properties(inner, "Properties", QString(), "property", "properties");
+            FastSection relatedNonMembers(inner,
                                           "Related Non-Members",
                                           QString(),
                                           "related non-member",
                                           "related non-members");
-            FastSection staticPrivateMembers(classNode,
+            FastSection staticPrivateMembers(inner,
                                              "Static Private Members",
                                              QString(),
                                              "static private member",
                                              "static private members");
-            FastSection staticProtectedMembers(classNode,
+            FastSection staticProtectedMembers(inner,
                                                "Static Protected Members",
                                                QString(),
                                                "static protected member",
                                                "static protected members");
-            FastSection staticPublicMembers(classNode,
+            FastSection staticPublicMembers(inner,
                                             "Static Public Members",
                                             QString(),
                                             "static public member",
                                             "static public members");
             FastSection macros(inner, "Macros", QString(), "macro", "macros");
 
-            NodeList::ConstIterator r = classNode->relatedNodes().constBegin();
-            while (r != classNode->relatedNodes().constEnd()) {
+            NodeList::ConstIterator r = inner->relatedNodes().constBegin();
+            while (r != inner->relatedNodes().constEnd()) {
                 if ((*r)->type() == Node::Function) {
                     FunctionNode *func = static_cast<FunctionNode *>(*r);
                     if (func->isMacro())
@@ -524,13 +522,13 @@ QList<Section> CppCodeMarker::sections(const InnerNode *inner,
                 ++r;
             }
 
-            QStack<const ClassNode *> stack;
-            stack.push(classNode);
+            QStack<const InnerNode *> stack;
+            stack.push(inner);
             while (!stack.isEmpty()) {
-                const ClassNode *ancestorClass = stack.pop();
+                const InnerNode* ancestor = stack.pop();
 
-                NodeList::ConstIterator c = ancestorClass->childNodes().constBegin();
-                while (c != ancestorClass->childNodes().constEnd()) {
+                NodeList::ConstIterator c = ancestor->childNodes().constBegin();
+                while (c != ancestor->childNodes().constEnd()) {
                     bool isSlot = false;
                     bool isSignal = false;
                     bool isStatic = false;
@@ -620,15 +618,16 @@ QList<Section> CppCodeMarker::sections(const InnerNode *inner,
                     ++c;
                 }
 
-                QList<RelatedClass>::ConstIterator r =
-                        ancestorClass->baseClasses().constBegin();
-                while (r != ancestorClass->baseClasses().constEnd()) {
-                    if ((*r).node_)
-                        stack.prepend((*r).node_);
-                    ++r;
+                if (ancestor->isClass()) {
+                    const ClassNode* cn = static_cast<const ClassNode*>(ancestor);
+                    QList<RelatedClass>::ConstIterator r = cn->baseClasses().constBegin();
+                    while (r != cn->baseClasses().constEnd()) {
+                        if ((*r).node_)
+                            stack.prepend((*r).node_);
+                        ++r;
+                    }
                 }
             }
-
             append(sections, publicTypes);
             append(sections, properties);
             append(sections, publicFunctions);
@@ -649,15 +648,15 @@ QList<Section> CppCodeMarker::sections(const InnerNode *inner,
             append(sections, macros);
         }
         else if (style == Detailed) {
-            FastSection memberFunctions(classNode,"Member Function Documentation","func","member","members");
-            FastSection memberTypes(classNode,"Member Type Documentation","types","member","members");
-            FastSection memberVariables(classNode,"Member Variable Documentation","vars","member","members");
-            FastSection properties(classNode,"Property Documentation","prop","member","members");
-            FastSection relatedNonMembers(classNode,"Related Non-Members","relnonmem","member","members");
-            FastSection macros(classNode,"Macro Documentation","macros","member","members");
-
-            NodeList::ConstIterator r = classNode->relatedNodes().constBegin();
-            while (r != classNode->relatedNodes().constEnd()) {
+            FastSection memberFunctions(inner,"Member Function Documentation","func","member","members");
+            FastSection memberTypes(inner,"Member Type Documentation","types","member","members");
+            FastSection memberVariables(inner,"Member Variable Documentation","vars","member","members");
+            FastSection properties(inner,"Property Documentation","prop","member","members");
+            FastSection relatedNonMembers(inner,"Related Non-Members","relnonmem","member","members");
+            FastSection macros(inner,"Macro Documentation","macros","member","members");
+
+            NodeList::ConstIterator r = inner->relatedNodes().constBegin();
+            while (r != inner->relatedNodes().constEnd()) {
                 if ((*r)->type() == Node::Function) {
                     FunctionNode *func = static_cast<FunctionNode *>(*r);
                     if (func->isMacro())
@@ -671,8 +670,8 @@ QList<Section> CppCodeMarker::sections(const InnerNode *inner,
                 ++r;
             }
 
-            NodeList::ConstIterator c = classNode->childNodes().constBegin();
-            while (c != classNode->childNodes().constEnd()) {
+            NodeList::ConstIterator c = inner->childNodes().constBegin();
+            while (c != inner->childNodes().constEnd()) {
                 if ((*c)->type() == Node::Enum ||
                         (*c)->type() == Node::Typedef) {
                     insert(memberTypes, *c, style, status);
@@ -700,28 +699,28 @@ QList<Section> CppCodeMarker::sections(const InnerNode *inner,
             append(sections, macros);
         }
         else {
-            FastSection all(classNode,QString(),QString(),"member","members");
+            FastSection all(inner,QString(),QString(),"member","members");
 
-            QStack<const ClassNode *> stack;
-            stack.push(classNode);
+            QStack<const InnerNode*> stack;
+            stack.push(inner);
 
             while (!stack.isEmpty()) {
-                const ClassNode *ancestorClass = stack.pop();
-
-                NodeList::ConstIterator c = ancestorClass->childNodes().constBegin();
-                while (c != ancestorClass->childNodes().constEnd()) {
-                    if ((*c)->access() != Node::Private &&
-                            (*c)->type() != Node::Property)
+                const InnerNode* ancestor = stack.pop();
+                NodeList::ConstIterator c = ancestor->childNodes().constBegin();
+                while (c != ancestor->childNodes().constEnd()) {
+                    if ((*c)->access() != Node::Private && (*c)->type() != Node::Property)
                         insert(all, *c, style, status);
                     ++c;
                 }
 
-                QList<RelatedClass>::ConstIterator r =
-                        ancestorClass->baseClasses().constBegin();
-                while (r != ancestorClass->baseClasses().constEnd()) {
-                    if ((*r).node_)
-                        stack.prepend((*r).node_);
-                    ++r;
+                if (ancestor->isClass()) {
+                    const ClassNode* cn = static_cast<const ClassNode*>(ancestor);
+                    QList<RelatedClass>::ConstIterator r = cn->baseClasses().constBegin();
+                    while (r != cn->baseClasses().constEnd()) {
+                        if ((*r).node_)
+                            stack.prepend((*r).node_);
+                        ++r;
+                    }
                 }
             }
             append(sections, all);
diff --git a/src/tools/qdoc/cppcodeparser.cpp b/src/tools/qdoc/cppcodeparser.cpp
index 6cb86a7515..bc24ff5daa 100644
--- a/src/tools/qdoc/cppcodeparser.cpp
+++ b/src/tools/qdoc/cppcodeparser.cpp
@@ -423,6 +423,10 @@ Node* CppCodeParser::processTopicCommand(const Doc& doc,
 
         }
         else if (node->isInnerNode()) {
+            if (type == Node::Namespace) {
+                NamespaceNode* ns = static_cast<NamespaceNode*>(node);
+                ns->markSeen();
+            }
             /*
               This treats a class as a namespace.
              */
diff --git a/src/tools/qdoc/htmlgenerator.cpp b/src/tools/qdoc/htmlgenerator.cpp
index 8eb96bff17..7518b20fa3 100644
--- a/src/tools/qdoc/htmlgenerator.cpp
+++ b/src/tools/qdoc/htmlgenerator.cpp
@@ -291,7 +291,12 @@ void HtmlGenerator::generateKeywordAnchors(const Node* node)
 }
 
 /*!
-  Traverses the current tree generating all the HTML documentation.
+  If qdoc is in the \c {-prepare} phase, traverse the primary
+  tree to generate the index file for the current module.
+
+  If qdoc is in the \c {-generate} phase, traverse the primary
+  tree to generate all the HTML documentation for the current
+  module. Then generate the help file and the tag file.
  */
 void HtmlGenerator::generateDocs()
 {
diff --git a/src/tools/qdoc/node.cpp b/src/tools/qdoc/node.cpp
index e14ca4af3e..25792a7c35 100644
--- a/src/tools/qdoc/node.cpp
+++ b/src/tools/qdoc/node.cpp
@@ -1112,7 +1112,9 @@ bool InnerNode::isSameSignature(const FunctionNode *f1, const FunctionNode *f2)
 }
 
 /*!
-  Adds the \a child to this node's child list.
+  Adds the \a child to this node's child list. It might also
+  be necessary to update this node's internal collections and
+  the child's parent pointer and output subdirectory.
  */
 void InnerNode::addChild(Node *child)
 {
@@ -1132,6 +1134,10 @@ void InnerNode::addChild(Node *child)
             enumChildren_.append(child);
         childMap.insertMulti(child->name(), child);
     }
+    if (child->parent() == 0) {
+        child->setParent(this);
+        child->setOutputSubdirectory(this->outputSubdirectory());
+    }
 }
 
 /*!
@@ -1146,6 +1152,10 @@ void InnerNode::addChild(Node* child, const QString& title)
 }
 
 /*!
+  The \a child is removed from this node's child list and
+  from this node's internal collections. The child's parent
+  pointer is set to 0, but its output subdirectory is not
+  changed.
  */
 void InnerNode::removeChild(Node *child)
 {
@@ -1185,6 +1195,7 @@ void InnerNode::removeChild(Node *child)
         }
         ++ent;
     }
+    child->setParent(0);
 }
 
 /*!
@@ -1357,7 +1368,7 @@ LeafNode::LeafNode(InnerNode* parent, Type type, const QString& name)
   Constructs a namespace node.
  */
 NamespaceNode::NamespaceNode(InnerNode *parent, const QString& name)
-    : InnerNode(Namespace, parent, name), tree_(0)
+    : InnerNode(Namespace, parent, name), seen_(false), tree_(0)
 {
     setGenus(Node::CPP);
     setPageType(ApiPage);
diff --git a/src/tools/qdoc/node.h b/src/tools/qdoc/node.h
index b6717b3b1b..5eeb7e7d3d 100644
--- a/src/tools/qdoc/node.h
+++ b/src/tools/qdoc/node.h
@@ -373,9 +373,7 @@ public:
     virtual ~InnerNode();
 
     Node* findChildNode(const QString& name, Node::Genus genus) const;
-    //Node* findChildNode(const QString& name, bool qml) const;
     Node* findChildNode(const QString& name, Type type);
-    //void findNodes(const QString& name, NodeList& n);
     virtual void findChildren(const QString& name, NodeList& nodes) const Q_DECL_OVERRIDE;
     FunctionNode* findFunctionNode(const QString& name) const;
     FunctionNode* findFunctionNode(const FunctionNode* clone);
@@ -410,6 +408,8 @@ public:
     const QStringList& groupNames() const { return groupNames_; }
     virtual void appendGroupName(const QString& t) Q_DECL_OVERRIDE { groupNames_.append(t); }
     void printChildren(const QString& title);
+    void addChild(Node* child);
+    void removeChild(Node* child);
 
 protected:
     InnerNode(Type type, InnerNode* parent, const QString& name);
@@ -418,9 +418,7 @@ private:
     friend class Node;
 
     static bool isSameSignature(const FunctionNode* f1, const FunctionNode* f2);
-    void addChild(Node* child);
     void removeRelated(Node* pseudoChild);
-    void removeChild(Node* child);
 
     QString outputFileName_;
     QStringList pageKeywds;
@@ -455,9 +453,14 @@ public:
     virtual ~NamespaceNode() { }
     virtual bool isNamespace() const Q_DECL_OVERRIDE { return true; }
     virtual Tree* tree() const Q_DECL_OVERRIDE { return (parent() ? parent()->tree() : tree_); }
+    virtual bool wasSeen() const Q_DECL_OVERRIDE { return seen_; }
+
+    void markSeen() { seen_ = true; }
+    void markNotSeen() { seen_ = false; }
     void setTree(Tree* t) { tree_ = t; }
 
  private:
+    bool        seen_;
     Tree*       tree_;
 };
 
diff --git a/src/tools/qdoc/qdocdatabase.cpp b/src/tools/qdoc/qdocdatabase.cpp
index 75295613f7..30d4d28a17 100644
--- a/src/tools/qdoc/qdocdatabase.cpp
+++ b/src/tools/qdoc/qdocdatabase.cpp
@@ -796,7 +796,6 @@ void QDocDatabase::processForest()
 {
     Tree* t = forest_.firstTree();
     while (t) {
-        findAllNamespaces(t->root());
         findAllClasses(t->root());
         findAllFunctions(t->root());
         findAllObsoleteThings(t->root());
@@ -805,6 +804,7 @@ void QDocDatabase::processForest()
         t->setTreeHasBeenAnalyzed();
         t = forest_.nextTree();
     }
+    resolveNamespaces();
 }
 
 /*!
@@ -872,16 +872,10 @@ NodeMap& QDocDatabase::getQmlTypesWithObsoleteMembers()
     return qmlTypesWithObsoleteMembers_;
 }
 
-/*!
-  Constructs the C++ namespace data structure, if it has not
-  already been constructed. Returns a reference to it.
+/*! \fn NodeMap& QDocDatabase::getNamespaces()
+  Returns a reference to the map of all namespace nodes.
+  This function must not be called in the -prepare phase.
  */
-NodeMap& QDocDatabase::getNamespaces()
-{
-    if (namespaceIndex_.isEmpty())
-        processForest(&QDocDatabase::findAllNamespaces);
-    return namespaceIndex_;
-}
 
 /*!
   Construct the C++ class data structures, if they have not
@@ -1082,14 +1076,15 @@ void QDocDatabase::findAllNamespaces(InnerNode* node)
 {
     NodeList::ConstIterator c = node->childNodes().constBegin();
     while (c != node->childNodes().constEnd()) {
-        if ((*c)->access() != Node::Private) {
+        if ((*c)->access() != Node::Private || (*c)->isNamespace()) {
             if ((*c)->isInnerNode()) {
                 findAllNamespaces(static_cast<InnerNode *>(*c));
-                if ((*c)->type() == Node::Namespace) {
+                if ((*c)->isNamespace()) {
                     // Ensure that the namespace's name is not empty (the root
                     // namespace has no name).
-                    if (!(*c)->name().isEmpty())
-                        namespaceIndex_.insert((*c)->name(), *c);
+                    if (!(*c)->name().isEmpty()) {
+                        nmm_.insert((*c)->name(), *c);
+                    }
                 }
             }
         }
@@ -1332,8 +1327,56 @@ void QDocDatabase::resolveStuff()
     //primaryTree()->resolveTargets(primaryTreeRoot());
     primaryTree()->resolveCppToQmlLinks();
     primaryTree()->resolveUsingClauses();
+    resolveNamespaces();
+}
+
+/*!
+ */
+void QDocDatabase::resolveNamespaces()
+{
+    if (!namespaceIndex_.isEmpty())
+        return;
+    Tree* t = forest_.firstTree();
+    while (t) {
+        findAllNamespaces(t->root());
+        t = forest_.nextTree();
+    }
+    QList<QString> keys = nmm_.uniqueKeys();
+    foreach (QString s, keys) {
+        NamespaceNode* ns = 0;
+        QList<Node*> nodes = nmm_.values(s);
+        int count = nmm_.remove(s);
+        if (count > 1) {
+            foreach (Node* n, nodes) {
+                if (n->isNamespace() && n->wasSeen()) {
+                    ns = static_cast<NamespaceNode*>(n);
+                    break;
+                }
+            }
+        }
+        else if (count == 1)
+            ns = static_cast<NamespaceNode*>(nodes.at(0));
+        if (ns && ns->wasSeen()) {
+            if (count >1) {
+                foreach (Node* n, nodes) {
+                    if (n->isNamespace()) {
+                        NamespaceNode* NS = static_cast<NamespaceNode*>(n);
+                        if (NS != ns) {
+                            while (!NS->childNodes().isEmpty()) {
+                                Node* child = NS->childNodes().first();
+                                NS->removeChild(child);
+                                ns->addChild(child);
+                            }
+                        }
+                    }
+                }
+            }
+            namespaceIndex_.insert(ns->name(), ns);
+        }
+    }
 }
 
+
 /*!
   This function is called for autolinking to a \a type,
   which could be a function return type or a parameter
diff --git a/src/tools/qdoc/qdocdatabase.h b/src/tools/qdoc/qdocdatabase.h
index 70307aa3ae..d0c59d731c 100644
--- a/src/tools/qdoc/qdocdatabase.h
+++ b/src/tools/qdoc/qdocdatabase.h
@@ -278,7 +278,7 @@ class QDocDatabase
     NodeMap& getClassesWithObsoleteMembers();
     NodeMap& getObsoleteQmlTypes();
     NodeMap& getQmlTypesWithObsoleteMembers();
-    NodeMap& getNamespaces();
+    NodeMap& getNamespaces() { resolveNamespaces(); return namespaceIndex_; }
     NodeMap& getServiceClasses();
     NodeMap& getQmlBasicTypes();
     NodeMap& getQmlTypes();
@@ -404,9 +404,8 @@ class QDocDatabase
     }
     TargetList* getTargetList(const QString& t) { return primaryTree()->getTargetList(t); }
     QStringList getTargetListKeys() { return primaryTree()->getTargetListKeys(); }
-    QStringList keys() {
-        return forest_.keys();
-    }
+    QStringList keys() { return forest_.keys(); }
+    void resolveNamespaces();
 
  private:
     friend class QDocIndexFiles;
@@ -447,6 +446,7 @@ class QDocDatabase
     NodeMap                 obsoleteQmlTypes_;
     NodeMap                 qmlTypesWithObsoleteMembers_;
     NodeMap                 namespaceIndex_;
+    NodeMultiMap            nmm_;
     NodeMap                 serviceClasses_; // MWS: not needed, should be deleted
     NodeMap                 qmlBasicTypes_;
     NodeMap                 qmlTypes_;