qdoc: Fixed pure doc parser

This parser was meant to parse any file for
qdoc comments only, ignoring everything else
that is not inside a qdoc comment. But it
wasn't doing that. It was parsing all code,
regardless of the language, using the C++
parser. Now it has been corrected to look
at qdoc comments and skip over everything
else.

Note thast this means qdoc will expect a
qdoc topic command in each and every qdoc
comment in the file. The posiution of the
qdoc comment with respect to the code it is
meant to document is not taken into account
in the pure doc parser. This is in contrast
to the QML and C++ parsers which do take
comment location into account in some cases.

Change-Id: I0804a4149baa942b463e0b6990c71e4039ac1a50
Reviewed-by: Keith Isdale <keith.isdale@nokia.com>
Reviewed-by: Martin Smith <martin.smith@nokia.com>

7 files changed, 213 insertions, 60 deletions

diff --git a/src/tools/qdoc/cppcodeparser.cpp b/src/tools/qdoc/cppcodeparser.cpp
index b5dcae271c..e7c75956df 100644
--- a/src/tools/qdoc/cppcodeparser.cpp
+++ b/src/tools/qdoc/cppcodeparser.cpp
@@ -57,60 +57,6 @@ QT_BEGIN_NAMESPACE
 
 /* qmake ignore Q_OBJECT */
 
-#define COMMAND_CLASS                   Doc::alias("class")
-#define COMMAND_CONTENTSPAGE            Doc::alias("contentspage")
-#define COMMAND_DITAMAP                 Doc::alias("ditamap")
-#define COMMAND_ENUM                    Doc::alias("enum")
-#define COMMAND_EXAMPLE                 Doc::alias("example")
-#define COMMAND_EXTERNALPAGE            Doc::alias("externalpage")
-#define COMMAND_FILE                    Doc::alias("file")
-#define COMMAND_FN                      Doc::alias("fn")
-#define COMMAND_GROUP                   Doc::alias("group")
-#define COMMAND_HEADERFILE              Doc::alias("headerfile")
-#define COMMAND_INDEXPAGE               Doc::alias("indexpage")
-#define COMMAND_INHEADERFILE            Doc::alias("inheaderfile")
-#define COMMAND_MACRO                   Doc::alias("macro")
-#define COMMAND_MODULE                  Doc::alias("module")
-#define COMMAND_NAMESPACE               Doc::alias("namespace")
-#define COMMAND_OVERLOAD                Doc::alias("overload")
-#define COMMAND_NEXTPAGE                Doc::alias("nextpage")
-#define COMMAND_PAGE                    Doc::alias("page")
-#define COMMAND_PREVIOUSPAGE            Doc::alias("previouspage")
-#define COMMAND_PROPERTY                Doc::alias("property")
-#define COMMAND_REIMP                   Doc::alias("reimp")
-#define COMMAND_RELATES                 Doc::alias("relates")
-#define COMMAND_STARTPAGE               Doc::alias("startpage")
-#define COMMAND_TYPEDEF                 Doc::alias("typedef")
-#define COMMAND_VARIABLE                Doc::alias("variable")
-#define COMMAND_QMLABSTRACT             Doc::alias("qmlabstract")
-#define COMMAND_QMLCLASS                Doc::alias("qmlclass")
-#define COMMAND_QMLPROPERTY             Doc::alias("qmlproperty")
-#define COMMAND_QMLATTACHEDPROPERTY     Doc::alias("qmlattachedproperty")
-#define COMMAND_QMLINHERITS             Doc::alias("inherits")
-#define COMMAND_QMLSIGNAL               Doc::alias("qmlsignal")
-#define COMMAND_QMLATTACHEDSIGNAL       Doc::alias("qmlattachedsignal")
-#define COMMAND_QMLMETHOD               Doc::alias("qmlmethod")
-#define COMMAND_QMLATTACHEDMETHOD       Doc::alias("qmlattachedmethod")
-#define COMMAND_QMLDEFAULT              Doc::alias("default")
-#define COMMAND_QMLREADONLY             Doc::alias("readonly")
-#define COMMAND_QMLBASICTYPE            Doc::alias("qmlbasictype")
-#define COMMAND_QMLMODULE               Doc::alias("qmlmodule")
-#define COMMAND_AUDIENCE                Doc::alias("audience")
-#define COMMAND_CATEGORY                Doc::alias("category")
-#define COMMAND_PRODNAME                Doc::alias("prodname")
-#define COMMAND_COMPONENT               Doc::alias("component")
-#define COMMAND_AUTHOR                  Doc::alias("author")
-#define COMMAND_PUBLISHER               Doc::alias("publisher")
-#define COMMAND_COPYRYEAR               Doc::alias("copyryear")
-#define COMMAND_COPYRHOLDER             Doc::alias("copyrholder")
-#define COMMAND_PERMISSIONS             Doc::alias("permissions")
-#define COMMAND_LIFECYCLEVERSION        Doc::alias("lifecycleversion")
-#define COMMAND_LIFECYCLEWSTATUS        Doc::alias("lifecyclestatus")
-#define COMMAND_LICENSEYEAR             Doc::alias("licenseyear")
-#define COMMAND_LICENSENAME             Doc::alias("licensename")
-#define COMMAND_LICENSEDESCRIPTION      Doc::alias("licensedescription")
-#define COMMAND_RELEASEDATE             Doc::alias("releasedate")
-
 QStringList CppCodeParser::exampleFiles;
 QStringList CppCodeParser::exampleDirs;
 
diff --git a/src/tools/qdoc/cppcodeparser.h b/src/tools/qdoc/cppcodeparser.h
index 6ae7116d9e..45a6c466ec 100644
--- a/src/tools/qdoc/cppcodeparser.h
+++ b/src/tools/qdoc/cppcodeparser.h
@@ -106,12 +106,14 @@ protected:
                                          Node *node);
     void processOtherMetaCommands(const Doc& doc, Node *node);
 
-private:
+ protected:
     void reset(Tree *tree);
     void readToken();
     const Location& location();
     QString previousLexeme();
     QString lexeme();
+
+ private:
     bool match(int target);
     bool skipTo(int target);
     bool matchCompat();
@@ -157,6 +159,7 @@ private:
                                   Tree *tree);
     void createExampleFileNodes(FakeNode *fake);
 
+ protected:
     QMap<QString, Node::Type> nodeTypeMap;
     Tree* tree_;
     Tokenizer *tokenizer;
@@ -167,12 +170,13 @@ private:
     QStringList lastPath;
     QRegExp varComment;
     QRegExp sep;
+    QSet<QString> activeNamespaces_;
 
+ private:
     QString sequentialIteratorDefinition;
     QString mutableSequentialIteratorDefinition;
     QString associativeIteratorDefinition;
     QString mutableAssociativeIteratorDefinition;
-    QSet<QString> activeNamespaces_;
     QMap<QString, QString> sequentialIteratorClasses;
     QMap<QString, QString> mutableSequentialIteratorClasses;
     QMap<QString, QString> associativeIteratorClasses;
@@ -184,6 +188,60 @@ private:
     QString exampleImageFilter;
 };
 
+#define COMMAND_CLASS                   Doc::alias("class")
+#define COMMAND_CONTENTSPAGE            Doc::alias("contentspage")
+#define COMMAND_DITAMAP                 Doc::alias("ditamap")
+#define COMMAND_ENUM                    Doc::alias("enum")
+#define COMMAND_EXAMPLE                 Doc::alias("example")
+#define COMMAND_EXTERNALPAGE            Doc::alias("externalpage")
+#define COMMAND_FILE                    Doc::alias("file")
+#define COMMAND_FN                      Doc::alias("fn")
+#define COMMAND_GROUP                   Doc::alias("group")
+#define COMMAND_HEADERFILE              Doc::alias("headerfile")
+#define COMMAND_INDEXPAGE               Doc::alias("indexpage")
+#define COMMAND_INHEADERFILE            Doc::alias("inheaderfile")
+#define COMMAND_MACRO                   Doc::alias("macro")
+#define COMMAND_MODULE                  Doc::alias("module")
+#define COMMAND_NAMESPACE               Doc::alias("namespace")
+#define COMMAND_OVERLOAD                Doc::alias("overload")
+#define COMMAND_NEXTPAGE                Doc::alias("nextpage")
+#define COMMAND_PAGE                    Doc::alias("page")
+#define COMMAND_PREVIOUSPAGE            Doc::alias("previouspage")
+#define COMMAND_PROPERTY                Doc::alias("property")
+#define COMMAND_REIMP                   Doc::alias("reimp")
+#define COMMAND_RELATES                 Doc::alias("relates")
+#define COMMAND_STARTPAGE               Doc::alias("startpage")
+#define COMMAND_TYPEDEF                 Doc::alias("typedef")
+#define COMMAND_VARIABLE                Doc::alias("variable")
+#define COMMAND_QMLABSTRACT             Doc::alias("qmlabstract")
+#define COMMAND_QMLCLASS                Doc::alias("qmlclass")
+#define COMMAND_QMLPROPERTY             Doc::alias("qmlproperty")
+#define COMMAND_QMLATTACHEDPROPERTY     Doc::alias("qmlattachedproperty")
+#define COMMAND_QMLINHERITS             Doc::alias("inherits")
+#define COMMAND_QMLSIGNAL               Doc::alias("qmlsignal")
+#define COMMAND_QMLATTACHEDSIGNAL       Doc::alias("qmlattachedsignal")
+#define COMMAND_QMLMETHOD               Doc::alias("qmlmethod")
+#define COMMAND_QMLATTACHEDMETHOD       Doc::alias("qmlattachedmethod")
+#define COMMAND_QMLDEFAULT              Doc::alias("default")
+#define COMMAND_QMLREADONLY             Doc::alias("readonly")
+#define COMMAND_QMLBASICTYPE            Doc::alias("qmlbasictype")
+#define COMMAND_QMLMODULE               Doc::alias("qmlmodule")
+#define COMMAND_AUDIENCE                Doc::alias("audience")
+#define COMMAND_CATEGORY                Doc::alias("category")
+#define COMMAND_PRODNAME                Doc::alias("prodname")
+#define COMMAND_COMPONENT               Doc::alias("component")
+#define COMMAND_AUTHOR                  Doc::alias("author")
+#define COMMAND_PUBLISHER               Doc::alias("publisher")
+#define COMMAND_COPYRYEAR               Doc::alias("copyryear")
+#define COMMAND_COPYRHOLDER             Doc::alias("copyrholder")
+#define COMMAND_PERMISSIONS             Doc::alias("permissions")
+#define COMMAND_LIFECYCLEVERSION        Doc::alias("lifecycleversion")
+#define COMMAND_LIFECYCLEWSTATUS        Doc::alias("lifecyclestatus")
+#define COMMAND_LICENSEYEAR             Doc::alias("licenseyear")
+#define COMMAND_LICENSENAME             Doc::alias("licensename")
+#define COMMAND_LICENSEDESCRIPTION      Doc::alias("licensedescription")
+#define COMMAND_RELEASEDATE             Doc::alias("releasedate")
+
 QT_END_NAMESPACE
 
 #endif
diff --git a/src/tools/qdoc/ditaxmlgenerator.cpp b/src/tools/qdoc/ditaxmlgenerator.cpp
index a955469c06..1b51ace58c 100644
--- a/src/tools/qdoc/ditaxmlgenerator.cpp
+++ b/src/tools/qdoc/ditaxmlgenerator.cpp
@@ -1406,7 +1406,7 @@ int DitaXmlGenerator::generateAtom(const Atom *atom,
                 /*
                   This attribute is not supported in DITA, and at the
                   moment, including it is causing a validation error
-                  wherever it is used. I think it is onlym used in the
+                  wherever it is used. I think it is only used in the
                   qdoc manual.
                  */
                 //xmlWriter().writeAttribute("start",atom->next()->string());
diff --git a/src/tools/qdoc/puredocparser.cpp b/src/tools/qdoc/puredocparser.cpp
index 072e6331a0..19a19373bd 100644
--- a/src/tools/qdoc/puredocparser.cpp
+++ b/src/tools/qdoc/puredocparser.cpp
@@ -43,6 +43,14 @@
   puredocparser.cpp
 */
 
+#include <qfile.h>
+#include <stdio.h>
+#include <errno.h>
+#include "codechunk.h"
+#include "config.h"
+#include "tokenizer.h"
+#include "tree.h"
+#include <qdebug.h>
 #include "puredocparser.h"
 
 QT_BEGIN_NAMESPACE
@@ -57,7 +65,143 @@ PureDocParser::~PureDocParser()
 
 QStringList PureDocParser::sourceFileNameFilter()
 {
-    return QStringList() << "*.qdoc" << "*.qtx" << "*.qtt";
+    return QStringList() << "*.qdoc" << "*.qtx" << "*.qtt" << "*.js";
 }
 
+/*!
+  Parse the source file identified by \a filePath and add its
+  parsed contents to the big \a tree. \a location is used for
+  reporting errors.
+ */
+void PureDocParser::parseSourceFile(const Location& location,
+                                    const QString& filePath,
+                                    Tree *tree)
+{
+    QFile in(filePath);
+    if (!in.open(QIODevice::ReadOnly)) {
+        location.error(tr("Can't open source file '%1' (%2)").arg(filePath).arg(strerror(errno)));
+        return;
+    }
+    createOutputSubdirectory(location, filePath);
+
+    reset(tree);
+    Location fileLocation(filePath);
+    Tokenizer fileTokenizer(fileLocation, in);
+    tokenizer = &fileTokenizer;
+    readToken();
+
+    /*
+      The set of active namespaces is cleared before parsing
+      each source file. The word "source" here means cpp file.
+     */
+    activeNamespaces_.clear();
+
+    processQdocComments();
+    in.close();
+}
+
+/*!
+  This is called by parseSourceFile() to do the actual parsing
+  and tree building. It only processes qdoc comments. It skips
+  everything else.
+ */
+bool PureDocParser::processQdocComments()
+{
+    QSet<QString> topicCommandsAllowed = topicCommands();
+    QSet<QString> otherMetacommandsAllowed = otherMetaCommands();
+    QSet<QString> metacommandsAllowed = topicCommandsAllowed + otherMetacommandsAllowed;
+
+    while (tok != Tok_Eoi) {
+        if (tok == Tok_Doc) {
+            /*
+              lexeme() returns an entire qdoc comment.
+             */
+            QString comment = lexeme();
+            Location start_loc(location());
+            readToken();
+
+            Doc::trimCStyleComment(start_loc,comment);
+            Location end_loc(location());
+
+            /*
+              Doc parses the comment.
+             */
+            Doc doc(start_loc,end_loc,comment,metacommandsAllowed);
+
+            QString topic;
+            ArgList args;
+
+            QSet<QString> topicCommandsUsed = topicCommandsAllowed & doc.metaCommandsUsed();
+
+            /*
+              There should be one topic command in the set,
+              or none. If the set is empty, then the comment
+              should be a function description.
+             */
+            if (topicCommandsUsed.count() > 0) {
+                topic = *topicCommandsUsed.begin();
+                args = doc.metaCommandArgs(topic);
+            }
+
+            NodeList nodes;
+            QList<Doc> docs;
+
+            if (topic.isEmpty()) {
+                doc.location().warning(tr("This qdoc comment contains no topic command "
+                                          "(e.g., '\\%1', '\\%2').")
+                                       .arg(COMMAND_MODULE).arg(COMMAND_PAGE));
+            }
+            else {
+                /*
+                  There is a topic command. Process it.
+                 */
+                if ((topic == COMMAND_QMLPROPERTY) ||
+                        (topic == COMMAND_QMLATTACHEDPROPERTY)) {
+                    Doc nodeDoc = doc;
+                    Node* node = processTopicCommandGroup(topic,args);
+                    if (node != 0) {
+                        nodes.append(node);
+                        docs.append(nodeDoc);
+                    }
+                }
+                else {
+                    ArgList::ConstIterator a = args.begin();
+                    while (a != args.end()) {
+                        Doc nodeDoc = doc;
+                        Node* node = processTopicCommand(nodeDoc,topic,*a);
+                        if (node != 0) {
+                            nodes.append(node);
+                            docs.append(nodeDoc);
+                        }
+                        ++a;
+                    }
+                }
+            }
+
+            NodeList::Iterator n = nodes.begin();
+            QList<Doc>::Iterator d = docs.begin();
+            while (n != nodes.end()) {
+                processOtherMetaCommands(*d, *n);
+                (*n)->setDoc(*d);
+                if ((*n)->isInnerNode() && ((InnerNode *)*n)->includes().isEmpty()) {
+                    InnerNode *m = static_cast<InnerNode *>(*n);
+                    while (m->parent() != tree_->root())
+                        m = m->parent();
+                    if (m == *n)
+                        ((InnerNode *)*n)->addInclude((*n)->name());
+                    else
+                        ((InnerNode *)*n)->setIncludes(m->includes());
+                }
+                ++d;
+                ++n;
+            }
+        }
+        else {
+            readToken();
+        }
+    }
+    return true;
+}
+
+
 QT_END_NAMESPACE
diff --git a/src/tools/qdoc/puredocparser.h b/src/tools/qdoc/puredocparser.h
index 80efadf6a4..085e6d85e6 100644
--- a/src/tools/qdoc/puredocparser.h
+++ b/src/tools/qdoc/puredocparser.h
@@ -65,6 +65,10 @@ public:
     virtual ~PureDocParser();
 
     virtual QStringList sourceFileNameFilter();
+    virtual void parseSourceFile(const Location& location, const QString& filePath, Tree* tree);
+
+ private:
+    bool processQdocComments();
 };
 
 QT_END_NAMESPACE
diff --git a/src/tools/qdoc/qmlcodeparser.cpp b/src/tools/qdoc/qmlcodeparser.cpp
index 23b8af8567..688087d793 100644
--- a/src/tools/qdoc/qmlcodeparser.cpp
+++ b/src/tools/qdoc/qmlcodeparser.cpp
@@ -126,7 +126,7 @@ QString QmlCodeParser::language()
  */
 QStringList QmlCodeParser::sourceFileNameFilter()
 {
-    return QStringList("*.qml");
+    return QStringList() << "*.qml";
 }
 
 /*!
diff --git a/src/tools/qdoc/tokenizer.cpp b/src/tools/qdoc/tokenizer.cpp
index e9906a13c5..82effb9bae 100644
--- a/src/tools/qdoc/tokenizer.cpp
+++ b/src/tools/qdoc/tokenizer.cpp
@@ -460,7 +460,8 @@ int Tokenizer::getToken()
             default:
                 // ### We should really prevent qdoc from looking at snippet files rather than
                 // ### suppress warnings when reading them.
-                if (yyNumPreprocessorSkipping == 0 && !yyTokLoc.fileName().endsWith(".qdoc")) {
+                if (yyNumPreprocessorSkipping == 0 && !(yyTokLoc.fileName().endsWith(".qdoc") ||
+                                                        yyTokLoc.fileName().endsWith(".js"))) {
                     yyTokLoc.warning(tr("Hostile character 0x%1 in C++ source")
                                      .arg((uchar)yyCh, 1, 16));
                 }