Add clang Tool and basic AST reading machinery

Also adds the possibility to indicate in the .pro file
or the command line option the path to the
compile_command.json file needed by clang
The path given to the command line has priority over
the one present in the .pro file

Change-Id: I5cddda7a7021d9c41cec98222c1ac42524d775f0
Reviewed-by: Karsten Heimrich <karsten.heimrich@qt.io>

10 files changed, 231 insertions, 22 deletions

diff --git a/src/linguist/lprodump/main.cpp b/src/linguist/lprodump/main.cpp
index 857786beb..33319c021 100644
--- a/src/linguist/lprodump/main.cpp
+++ b/src/linguist/lprodump/main.cpp
@@ -356,6 +356,11 @@ static QJsonArray processProjects(bool topLevel, const QStringList &proFiles,
                 tsFiles << proDir.filePath(tsFile);
             setValue(prj, "translations", tsFiles);
         }
+        if (visitor.contains(QLatin1String("LUPDATE_COMPILE_COMMANDS_PATH"))) {
+            const QStringList thepathjson = visitor.values(
+                QLatin1String("LUPDATE_COMPILE_COMMANDS_PATH"));
+            setValue(prj, "compileCommands", thepathjson.value(0));
+        }
         result.append(prj);
         pro->deref();
     }
diff --git a/src/linguist/lupdate/clangtoolastreader.h b/src/linguist/lupdate/clangtoolastreader.h
new file mode 100644
index 000000000..e43bc471d
--- /dev/null
+++ b/src/linguist/lupdate/clangtoolastreader.h
@@ -0,0 +1,134 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt Linguist of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef CLANG_TOOL_AST_READER_H
+#define CLANG_TOOL_AST_READER_H
+
+#include "lupdate.h"
+#include "translator.h"
+#include "translatormessage.h"
+
+#include <QtCore/qloggingcategory.h>
+
+#if defined(Q_CC_MSVC)
+# pragma warning(disable: 4100)
+# pragma warning(disable: 4146)
+# pragma warning(disable: 4267)
+# pragma warning(disable: 4624)
+#endif
+
+#include <llvm/ADT/APInt.h>
+#include <clang/AST/RecursiveASTVisitor.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/FrontendActions.h>
+#include <clang/Tooling/Tooling.h>
+
+#if defined(Q_CC_MSVC)
+# pragma warning(default: 4100)
+# pragma warning(default: 4146)
+# pragma warning(default: 4267)
+# pragma warning(default: 4624)
+#endif
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+QT_BEGIN_NAMESPACE
+
+Q_DECLARE_LOGGING_CATEGORY(lcClang)
+
+class LupdateVisitor : public clang::RecursiveASTVisitor<LupdateVisitor>
+{
+public:
+    explicit LupdateVisitor(clang::ASTContext *Context, Translator *tor)
+        : m_tor(tor),
+          m_context(Context)
+    {}
+
+private:
+    clang::ASTContext *m_context { nullptr };
+    Translator *m_tor { nullptr };
+};
+
+class LupdateASTConsumer : public clang::ASTConsumer
+{
+public:
+    explicit LupdateASTConsumer(clang::ASTContext *Context, Translator *tor)
+        : m_visitor(Context, tor)
+    {}
+
+    //This method is called when the ASTs for entire translation unit have been parsed.
+    void HandleTranslationUnit(clang::ASTContext &Context) override
+    {
+        bool traverse = m_visitor.TraverseAST(Context);
+        qCDebug(lcClang) << "TraverseAST: " << traverse << "\n";
+    }
+
+private:
+    LupdateVisitor m_visitor;
+};
+
+class LupdateFrontendAction : public clang::ASTFrontendAction
+{
+public:
+    LupdateFrontendAction(Translator *tor)
+        : m_tor(tor)
+    {}
+
+    std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
+        clang::CompilerInstance &Compiler, llvm::StringRef InFile) override
+    {
+        LupdateASTConsumer *consumer = new LupdateASTConsumer(&Compiler.getASTContext(), m_tor);
+        return std::unique_ptr<clang::ASTConsumer>(consumer);
+    }
+
+private:
+    Translator *m_tor { nullptr };
+};
+
+class LupdateToolActionFactory : public clang::tooling::FrontendActionFactory
+{
+public:
+    LupdateToolActionFactory(Translator *tor)
+        : m_tor(tor)
+    {}
+
+    clang::FrontendAction *create() override
+    {
+        return new LupdateFrontendAction(m_tor);
+    }
+
+private:
+    Translator *m_tor { nullptr };
+};
+
+QT_END_NAMESPACE
+
+#endif
diff --git a/src/linguist/lupdate/cpp_clang.cpp b/src/linguist/lupdate/cpp_clang.cpp
index 198724f50..598ac94ca 100644
--- a/src/linguist/lupdate/cpp_clang.cpp
+++ b/src/linguist/lupdate/cpp_clang.cpp
@@ -29,34 +29,65 @@
 
 #include <translator.h>
 
+#include <clang/Tooling/CommonOptionsParser.h>
+#include <llvm/Option/Option.h>
+
 QT_BEGIN_NAMESPACE
 
+Q_LOGGING_CATEGORY(lcClang, "qt.lupdate.clang");
+
+// This is a way to add options related to the customized clang tool
+// Needed as one of the arguments to create the OptionParser.
+static llvm::cl::OptionCategory MyToolCategory("my-tool options");
+
+// Makes sure all the comments will be parsed and part of the AST
+// Clang will run with the flag -fparse-all-comments
+clang::tooling::ArgumentsAdjuster getClangArgumentAdjuster()
+{
+    return [](const clang::tooling::CommandLineArguments &args, llvm::StringRef /*unused*/) {
+        clang::tooling::CommandLineArguments adjustedArgs;
+        for (size_t i = 0, e = args.size(); i < e; ++i) {
+            llvm::StringRef arg = args[i];
+            // FIXME: Remove options that generate output.
+            if (!arg.startswith("-fcolor-diagnostics") && !arg.startswith("-fdiagnostics-color"))
+                adjustedArgs.push_back(args[i]);
+        }
+        adjustedArgs.push_back("-fparse-all-comments");
+        adjustedArgs.push_back("-I");
+        adjustedArgs.push_back(CLANG_RESOURCE_DIR);
+        return adjustedArgs;
+    };
+}
+
 void ClangCppParser::loadCPP(Translator &translator, const QStringList &filenames,
     ConversionData &cd)
 {
-    // Going through the files to be parsed
     std::vector<std::string> sources;
-    for (const QString &filename: filenames)
-       sources.push_back(filename.toStdString());
+    for (const QString &filename : filenames)
+        sources.push_back(filename.toStdString());
 
     // The ClangTool is to be created and run from this function.
 
-    // First we'll need an OptionParser
-    // Then we'll create a ClangTool taking the OptionParser and the sources as argument
+    int argc = 4;
+    // NEED 2 empty one to start!!! otherwise: LLVM::ERROR
+    const QByteArray jsonPath = cd.m_compileCommandsPath.toLocal8Bit();
+    const char *argv[4] = { "", "", "-p", jsonPath.constData() };
+    clang::tooling::CommonOptionsParser OptionsParser(argc, argv, MyToolCategory);
+
+    clang::tooling::ClangTool tool(OptionsParser.getCompilations(), sources);
+    tool.appendArgumentsAdjuster(getClangArgumentAdjuster());
 
-    // The translator to store the information from the parsing of the files.
     Translator *tor = new Translator();
 
-    // TODO: set up clang tool for parsing
-    qWarning("lupdate: Clang based C++ parser not implemented!");
+    // A ClangTool needs a new FrontendAction for each translation unit it runs on
+    // A Customized FrontendActionFactory is building a customized FrondendAction
+    tool.run(new LupdateToolActionFactory(tor));
 
-    // TODO: remove this printing at a later point
-    // Printing the translator (storage and manipulation of translation info from linguist module)
-    if (qEnvironmentVariableIsSet("QT_LUPDATE_CLANG_DEBUG"))
+    if (QLoggingCategory("qt.lupdate.clang").isDebugEnabled())
         tor->dump();
 
-    for (const TranslatorMessage &msg: tor->messages())
+    for (const TranslatorMessage &msg : tor->messages())
         translator.extend(msg, cd);
-
 }
+
 QT_END_NAMESPACE
diff --git a/src/linguist/lupdate/cpp_clang.h b/src/linguist/lupdate/cpp_clang.h
index d4357f129..16e03fb70 100644
--- a/src/linguist/lupdate/cpp_clang.h
+++ b/src/linguist/lupdate/cpp_clang.h
@@ -29,14 +29,13 @@
 #ifndef CLANG_CPP_H
 #define CLANG_CPP_H
 
+#include "clangtoolastreader.h"
 #include "lupdate.h"
 
-
 QT_BEGIN_NAMESPACE
 
 namespace ClangCppParser {
-   void loadCPP(Translator &translator, const QStringList &filenames,
-   ConversionData &cd);
+    void loadCPP(Translator &translator, const QStringList &filenames, ConversionData &cd);
 }
 
 QT_END_NAMESPACE
diff --git a/src/linguist/lupdate/lupdate.h b/src/linguist/lupdate/lupdate.h
index 80076afbe..5dbd0e213 100644
--- a/src/linguist/lupdate/lupdate.h
+++ b/src/linguist/lupdate/lupdate.h
@@ -30,6 +30,7 @@
 #define LUPDATE_H
 
 #include "qglobal.h"
+#include <QtTools/private/qttools-config_p.h>
 
 #include <QtCore/QList>
 #include <QtCore/QHash>
diff --git a/src/linguist/lupdate/lupdate.pro b/src/linguist/lupdate/lupdate.pro
index 497a981e5..ddf115bab 100644
--- a/src/linguist/lupdate/lupdate.pro
+++ b/src/linguist/lupdate/lupdate.pro
@@ -18,6 +18,8 @@ qtConfig(clangcpp) {
     DEFINES += $$shell_quote(CLANG_RESOURCE_DIR=\"$${CLANG_LIBDIR}/clang/$${CLANG_VERSION}/include\")
 }
 
+CONFIG += rtti_off
+
 DEFINES += QT_NO_CAST_TO_ASCII QT_NO_CAST_FROM_ASCII
 
 include(../shared/formats.pri)
@@ -29,11 +31,9 @@ SOURCES += \
     ../shared/runqttool.cpp \
     ../shared/qrcreader.cpp \
     ../shared/simtexth.cpp \
-    \
     cpp.cpp \
     java.cpp \
-    ui.cpp \
-    cpp_clang.cpp
+    ui.cpp
 
 qtHaveModule(qmldevtools-private): SOURCES += qdeclarative.cpp
 
@@ -46,6 +46,14 @@ HEADERS += \
     ../shared/runqttool.h \
     ../shared/simtexth.h
 
+qtConfig(clangcpp) {
+    SOURCES += \
+        cpp_clang.cpp
+    HEADERS += \
+        cpp_clang.h \
+        clangtoolastreader.h
+}
+
 mingw {
     RC_FILE = lupdate.rc
 }
diff --git a/src/linguist/lupdate/main.cpp b/src/linguist/lupdate/main.cpp
index 5ab3f59b9..0c33308ef 100644
--- a/src/linguist/lupdate/main.cpp
+++ b/src/linguist/lupdate/main.cpp
@@ -28,7 +28,9 @@
 ****************************************************************************/
 
 #include "lupdate.h"
+#if QT_CONFIG(clangcpp)
 #include "cpp_clang.h"
+#endif
 
 #include <profileutils.h>
 #include <projectdescriptionreader.h>
@@ -49,6 +51,8 @@
 #include <iostream>
 
 bool useClangToParseCpp = false;
+QString commandLineCompileCommands; // for the path to the json file passed as a command line argument.
+                                    // Has priority over what is in the .pro file and passed to the project.
 
 // Can't have an array of QStaticStringData<N> for different N, so
 // use QString, which requires constructor calls. Doesn't matter
@@ -278,9 +282,15 @@ static void printUsage()
         "           Specify the output file(s). This will override the TRANSLATIONS.\n"
         "    -version\n"
         "           Display the version of lupdate and exit.\n"
+#if QT_CONFIG(clangcpp)
         "    -clang-parser \n"
         "           Use clang to parse cpp files. Otherwise a custom parser is used.\n"
         "           Need a compile_commands.json for the files that needs to be parsed.\n"
+        "           The path to this file can be given in the .pro file\n"
+        "           under LUPDATE_COMPILE_COMMANDS_PATH.\n"
+        "           If no path is given search for compile_commands.json will be attempted\n"
+        "           through all parent paths of the first input file.\n"
+#endif
         "    @lst-file\n"
         "           Read additional file names (one per line) or includepaths (one per\n"
         "           line, and prefixed with -I) from lst-file.\n"
@@ -519,8 +529,11 @@ static void processSources(Translator &fetchedTor,
         printErr(LU::tr("lupdate warning: Some files have been ignored due to missing qml/javascript support\n"));
 #endif
 
-    if (useClangToParseCpp)
+    if (useClangToParseCpp) {
+#if QT_CONFIG(clangcpp)
         ClangCppParser::loadCPP(fetchedTor, sourceFilesCpp, cd);
+#endif
+    }
     else
         loadCPP(fetchedTor, sourceFilesCpp, cd);
 
@@ -589,6 +602,10 @@ private:
         cd.m_includePath = prj.includePaths;
         cd.m_excludes = prj.excluded;
         cd.m_sourceIsUtf16 = options & SourceIsUtf16;
+        if (commandLineCompileCommands.isEmpty())
+            cd.m_compileCommandsPath = prj.compileCommands;
+        else
+            cd.m_compileCommandsPath = commandLineCompileCommands;
 
         QStringList tsFiles;
         if (hasTranslations(prj)) {
@@ -843,10 +860,19 @@ int main(int argc, char **argv)
                 includePath += args[i].mid(2);
             }
             continue;
-        } else if (arg == QLatin1String("-clang-parser")) {
+        }
+#if QT_CONFIG(clangcpp)
+        else if (arg == QLatin1String("-clang-parser")) {
             useClangToParseCpp = true;
+            // the option after -clang-parser is optional
+            if ((i + 1) != argc && !args[i + 1].startsWith(QLatin1String("-"))) {
+                 i++;
+                 commandLineCompileCommands = args[i];
+             }
             continue;
-        } else if (arg.startsWith(QLatin1String("-")) && arg != QLatin1String("-")) {
+        }
+#endif
+        else if (arg.startsWith(QLatin1String("-")) && arg != QLatin1String("-")) {
             printErr(LU::tr("Unrecognized option '%1'.\n").arg(arg));
             return 1;
         }
@@ -1008,6 +1034,7 @@ int main(int argc, char **argv)
         cd.m_projectRoots = projectRoots;
         cd.m_includePath = includePath;
         cd.m_allCSources = allCSources;
+        cd.m_compileCommandsPath = commandLineCompileCommands;
         for (const QString &resource : qAsConst(resourceFiles))
             sourceFiles << getResources(resource);
         processSources(fetchedTor, sourceFiles, cd);
diff --git a/src/linguist/shared/projectdescriptionreader.cpp b/src/linguist/shared/projectdescriptionreader.cpp
index e61b81e09..57fa39efc 100644
--- a/src/linguist/shared/projectdescriptionreader.cpp
+++ b/src/linguist/shared/projectdescriptionreader.cpp
@@ -70,6 +70,7 @@ private:
                     << QStringLiteral("excluded")
                     << QStringLiteral("includePaths")
                     << QStringLiteral("sources")
+                    << QStringLiteral("compileCommands")
                     << QStringLiteral("subProjects")
                     << QStringLiteral("translations");
         QSet<QString> actualKeys;
@@ -156,6 +157,7 @@ private:
         Project result;
         QJsonObject obj = v.toObject();
         result.filePath = stringValue(obj, QLatin1String("projectFile"));
+        result.compileCommands = stringValue(obj, QLatin1String("compileCommands"));
         result.codec = stringValue(obj, QLatin1String("codec"));
         result.excluded = stringListValue(obj, QLatin1String("excluded"));
         result.includePaths = stringListValue(obj, QLatin1String("includePaths"));
diff --git a/src/linguist/shared/projectdescriptionreader.h b/src/linguist/shared/projectdescriptionreader.h
index c7234d16b..f3d2ddf23 100644
--- a/src/linguist/shared/projectdescriptionreader.h
+++ b/src/linguist/shared/projectdescriptionreader.h
@@ -42,6 +42,7 @@ typedef std::vector<Project> Projects;
 struct Project
 {
     QString filePath;
+    QString compileCommands;
     QString codec;
     QStringList excluded;
     QStringList includePaths;
diff --git a/src/linguist/shared/translator.h b/src/linguist/shared/translator.h
index 5b5d12250..233e238d8 100644
--- a/src/linguist/shared/translator.h
+++ b/src/linguist/shared/translator.h
@@ -80,6 +80,7 @@ public:
     QString m_unTrPrefix; // QM specific
     QString m_sourceFileName;
     QString m_targetFileName;
+    QString m_compileCommandsPath;
     QStringList m_excludes;
     QDir m_sourceDir;
     QDir m_targetDir; // FIXME: TS specific