shiboken: No longer hard-code the C++ language level

The C++ language level was previously hard-coded in the default options.
This is potentially problematic for projects using shiboken and also
fell apart with Qt 5.12, where the experimental level "c++1z" used for
MSVC2017/Clang 4 no longer works due to not being able to handle
enumerator value deprecation attributes.

Introduce an enumeration to represent the level and add functions
to convert back to and forth to the respective Clang option.
Add an option to shiboken.

Add a function returning a default value for the emulated compiler,
returning C++ 14 or C++1Z for the CMSVC2017/Clang 4 case.

Task-number: PYSIDE-724
Change-Id: Ie7e19bf7f099a34e6cdaad4b462157a9a3ee8797
Reviewed-by: Christian Tismer <tismer@stackless.com>

11 files changed, 157 insertions, 14 deletions

diff --git a/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp b/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp
index 09d10580c..1d8a349f1 100644
--- a/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp
+++ b/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp
@@ -32,6 +32,7 @@
 
 #include <clangparser/clangbuilder.h>
 #include <clangparser/clangutils.h>
+#include <clangparser/compilersupport.h>
 
 #include "parser/codemodel.h"
 
@@ -443,10 +444,15 @@ void AbstractMetaBuilderPrivate::sortLists()
         cls->sortFunctions();
 }
 
-FileModelItem AbstractMetaBuilderPrivate::buildDom(const QByteArrayList &arguments,
+FileModelItem AbstractMetaBuilderPrivate::buildDom(QByteArrayList arguments,
+                                                   LanguageLevel level,
                                                    unsigned clangFlags)
 {
     clang::Builder builder;
+    if (level == LanguageLevel::Default)
+        level = clang::emulatedCompilerLanguageLevel();
+    arguments.prepend(QByteArrayLiteral("-std=")
+                      + clang::languageLevelOption(level));
     FileModelItem result = clang::parse(arguments, clangFlags, builder)
         ? builder.dom() : FileModelItem();
     const clang::BaseVisitor::Diagnostics &diagnostics = builder.diagnostics();
@@ -726,9 +732,11 @@ void AbstractMetaBuilderPrivate::traverseDom(const FileModelItem &dom)
     std::puts("");
 }
 
-bool AbstractMetaBuilder::build(const QByteArrayList &arguments, unsigned clangFlags)
+bool AbstractMetaBuilder::build(const QByteArrayList &arguments,
+                                LanguageLevel level,
+                                unsigned clangFlags)
 {
-    const FileModelItem dom = d->buildDom(arguments, clangFlags);
+    const FileModelItem dom = d->buildDom(arguments, level, clangFlags);
     if (dom.isNull())
         return false;
     if (ReportHandler::isDebug(ReportHandler::MediumDebug))
diff --git a/sources/shiboken2/ApiExtractor/abstractmetabuilder.h b/sources/shiboken2/ApiExtractor/abstractmetabuilder.h
index bd1236375..c8381d12b 100644
--- a/sources/shiboken2/ApiExtractor/abstractmetabuilder.h
+++ b/sources/shiboken2/ApiExtractor/abstractmetabuilder.h
@@ -32,6 +32,8 @@
 #include "abstractmetalang_typedefs.h"
 #include "dependency.h"
 
+#include "clangparser/compilersupport.h"
+
 QT_FORWARD_DECLARE_CLASS(QIODevice)
 
 class AbstractMetaBuilderPrivate;
@@ -71,7 +73,9 @@ public:
     AbstractMetaClassList classesTopologicalSorted(const AbstractMetaClass *cppClass = Q_NULLPTR,
                                                    const Dependencies &additionalDependencies = Dependencies()) const;
 
-    bool build(const QByteArrayList &arguments, unsigned clangFlags = 0);
+    bool build(const QByteArrayList &arguments,
+               LanguageLevel level = LanguageLevel::Default,
+               unsigned clangFlags = 0);
     void setLogDirectory(const QString& logDir);
 
     /**
diff --git a/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h b/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h
index 1f4e209db..6cbfd8d9d 100644
--- a/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h
+++ b/sources/shiboken2/ApiExtractor/abstractmetabuilder_p.h
@@ -46,7 +46,9 @@ public:
     AbstractMetaBuilderPrivate();
     ~AbstractMetaBuilderPrivate();
 
-    static FileModelItem buildDom(const QByteArrayList &arguments, unsigned clangFlags);
+    static FileModelItem buildDom(QByteArrayList arguments,
+                                  LanguageLevel level,
+                                  unsigned clangFlags);
     void traverseDom(const FileModelItem &dom);
 
     void dumpLog() const;
diff --git a/sources/shiboken2/ApiExtractor/apiextractor.cpp b/sources/shiboken2/ApiExtractor/apiextractor.cpp
index 4c4eaff6c..2dadda57a 100644
--- a/sources/shiboken2/ApiExtractor/apiextractor.cpp
+++ b/sources/shiboken2/ApiExtractor/apiextractor.cpp
@@ -259,8 +259,9 @@ bool ApiExtractor::run()
     for (const HeaderPath &headerPath : qAsConst(m_includePaths))
         arguments.append(HeaderPath::includeOption(headerPath));
     arguments.append(QFile::encodeName(preprocessedCppFileName));
-    qCDebug(lcShiboken) << __FUNCTION__ << arguments;
-    const bool result = m_builder->build(arguments);
+    qCDebug(lcShiboken) << __FUNCTION__ << arguments
+        << "level=" << int(m_languageLevel);
+    const bool result = m_builder->build(arguments, m_languageLevel);
     if (!result)
         autoRemove = false;
     if (!autoRemove) {
@@ -270,6 +271,16 @@ bool ApiExtractor::run()
     return result;
 }
 
+LanguageLevel ApiExtractor::languageLevel() const
+{
+    return m_languageLevel;
+}
+
+void ApiExtractor::setLanguageLevel(const LanguageLevel languageLevel)
+{
+    m_languageLevel = languageLevel;
+}
+
 #ifndef QT_NO_DEBUG_STREAM
 template <class Container>
 static void debugFormatSequence(QDebug &d, const char *key, const Container& c)
diff --git a/sources/shiboken2/ApiExtractor/apiextractor.h b/sources/shiboken2/ApiExtractor/apiextractor.h
index 702d3994c..af84223a6 100644
--- a/sources/shiboken2/ApiExtractor/apiextractor.h
+++ b/sources/shiboken2/ApiExtractor/apiextractor.h
@@ -36,6 +36,7 @@
 #include "header_paths.h"
 #include "typedatabase_typedefs.h"
 #include "typesystem_typedefs.h"
+#include "clangparser/compilersupport.h"
 #include <QStringList>
 
 class AbstractMetaBuilder;
@@ -75,6 +76,8 @@ public:
     void setLogDirectory(const QString& logDir);
     bool setApiVersion(const QString& package, const QString& version);
     void setDropTypeEntries(QString dropEntries);
+    LanguageLevel languageLevel() const;
+    void setLanguageLevel(const LanguageLevel languageLevel);
 
     AbstractMetaEnumList globalEnums() const;
     AbstractMetaFunctionList globalFunctions() const;
@@ -99,6 +102,7 @@ private:
     HeaderPaths m_includePaths;
     AbstractMetaBuilder* m_builder;
     QString m_logDirectory;
+    LanguageLevel m_languageLevel = LanguageLevel::Default;
 
     // disable copy
     ApiExtractor(const ApiExtractor&);
diff --git a/sources/shiboken2/ApiExtractor/clangparser/clangparser.cpp b/sources/shiboken2/ApiExtractor/clangparser/clangparser.cpp
index ddafcdb04..301b4211e 100644
--- a/sources/shiboken2/ApiExtractor/clangparser/clangparser.cpp
+++ b/sources/shiboken2/ApiExtractor/clangparser/clangparser.cpp
@@ -196,11 +196,6 @@ static CXTranslationUnit createTranslationUnit(CXIndex index,
         | CXTranslationUnit_Incomplete;
 
     static const QByteArrayList defaultArgs = {
-#if defined(Q_CC_MSVC) && _MSC_VER > 1900
-        "-std=c++1z", // Fixes constexpr errors in MSVC2017 library headers with Clang 4.1
-#else
-        "-std=c++14", // ! otherwise, t.h is parsed as "C"
-#endif
 #ifndef Q_OS_WIN
         "-fPIC",
 #endif
diff --git a/sources/shiboken2/ApiExtractor/clangparser/compilersupport.cpp b/sources/shiboken2/ApiExtractor/clangparser/compilersupport.cpp
index 9f4f7dc03..820909713 100644
--- a/sources/shiboken2/ApiExtractor/clangparser/compilersupport.cpp
+++ b/sources/shiboken2/ApiExtractor/clangparser/compilersupport.cpp
@@ -40,12 +40,19 @@
 #include <QtCore/QStringList>
 #include <QtCore/QVersionNumber>
 
+#include <clang-c/Index.h>
+
 #include <string.h>
 #include <algorithm>
 #include <iterator>
 
 namespace clang {
 
+QVersionNumber libClangVersion()
+{
+    return QVersionNumber(CINDEX_VERSION_MAJOR, CINDEX_VERSION_MINOR);
+}
+
 static bool runProcess(const QString &program, const QStringList &arguments,
                        QByteArray *stdOutIn = nullptr, QByteArray *stdErrIn = nullptr)
 {
@@ -271,4 +278,47 @@ QByteArrayList emulatedCompilerOptions()
     return result;
 }
 
+LanguageLevel emulatedCompilerLanguageLevel()
+{
+#if defined(Q_CC_MSVC) && _MSC_VER > 1900
+    // Fixes constexpr errors in MSVC2017 library headers with Clang 4.1..5.X (0.45 == Clang 6).
+    if (libClangVersion() < QVersionNumber(0, 45))
+        return LanguageLevel::Cpp1Z;
+#endif // Q_CC_MSVC && _MSC_VER > 1900
+    return LanguageLevel::Cpp14; // otherwise, t.h is parsed as "C"
+}
+
+struct LanguageLevelMapping
+{
+    const char *option;
+    LanguageLevel level;
+};
+
+static const LanguageLevelMapping languageLevelMapping[] =
+{
+    {"c++11", LanguageLevel::Cpp11},
+    {"c++14", LanguageLevel::Cpp14},
+    {"c++17", LanguageLevel::Cpp17},
+    {"c++20", LanguageLevel::Cpp20},
+    {"c++1z", LanguageLevel::Cpp1Z}
+};
+
+const char *languageLevelOption(LanguageLevel l)
+{
+    for (const LanguageLevelMapping &m : languageLevelMapping) {
+        if (m.level == l)
+            return m.option;
+    }
+    return nullptr;
+}
+
+LanguageLevel languageLevelFromOption(const char *o)
+{
+    for (const LanguageLevelMapping &m : languageLevelMapping) {
+        if (!strcmp(m.option, o))
+            return m.level;
+    }
+    return LanguageLevel::Default;
+}
+
 } // namespace clang
diff --git a/sources/shiboken2/ApiExtractor/clangparser/compilersupport.h b/sources/shiboken2/ApiExtractor/clangparser/compilersupport.h
index 68d09f6f5..d9e213e73 100644
--- a/sources/shiboken2/ApiExtractor/clangparser/compilersupport.h
+++ b/sources/shiboken2/ApiExtractor/clangparser/compilersupport.h
@@ -31,10 +31,25 @@
 
 #include <QtCore/QByteArrayList>
 
+QT_FORWARD_DECLARE_CLASS(QVersionNumber)
+
+enum class LanguageLevel {
+    Default,
+    Cpp11,
+    Cpp14,
+    Cpp17,
+    Cpp20,
+    Cpp1Z
+};
+
 namespace clang {
+QVersionNumber libClangVersion();
 
 QByteArrayList emulatedCompilerOptions();
+LanguageLevel emulatedCompilerLanguageLevel();
 
+const char *languageLevelOption(LanguageLevel l);
+LanguageLevel languageLevelFromOption(const char *);
 } // namespace clang
 
 #endif // COMPILERSUPPORT_H
diff --git a/sources/shiboken2/ApiExtractor/tests/testutil.h b/sources/shiboken2/ApiExtractor/tests/testutil.h
index 3d2dd19ea..6152793f5 100644
--- a/sources/shiboken2/ApiExtractor/tests/testutil.h
+++ b/sources/shiboken2/ApiExtractor/tests/testutil.h
@@ -67,7 +67,7 @@ namespace TestUtil
         tempSource.write(cppCode, qint64(strlen(cppCode)));
         tempSource.close();
         AbstractMetaBuilder *builder = new AbstractMetaBuilder;
-        if (!builder->build(arguments, 0)) {
+        if (!builder->build(arguments)) {
             delete builder;
             return Q_NULLPTR;
         }
diff --git a/sources/shiboken2/generator/main.cpp b/sources/shiboken2/generator/main.cpp
index b0ce849a7..7ee43710e 100644
--- a/sources/shiboken2/generator/main.cpp
+++ b/sources/shiboken2/generator/main.cpp
@@ -46,6 +46,7 @@
     #define PATH_SPLITTER ":"
 #endif
 
+static inline QString languageLevelOption() { return QStringLiteral("language-level"); }
 static inline QString includePathOption() { return QStringLiteral("include-paths"); }
 static inline QString frameworkIncludePathOption() { return QStringLiteral("framework-include-paths"); }
 static inline QString systemIncludePathOption() { return QStringLiteral("system-include-paths"); }
@@ -175,6 +176,7 @@ static bool processProjectFile(QFile& projectFile, QMap<QString, QString>& args)
     QStringList systemIncludePaths;
     QStringList typesystemPaths;
     QStringList apiVersions;
+    QString languageLevel;
 
     while (!projectFile.atEnd()) {
         line = projectFile.readLine().trimmed();
@@ -199,6 +201,8 @@ static bool processProjectFile(QFile& projectFile, QMap<QString, QString>& args)
             systemIncludePaths << QDir::toNativeSeparators(value);
         else if (key == "typesystem-path")
             typesystemPaths << QDir::toNativeSeparators(value);
+        else if (key == "language-level")
+            languageLevel = value;
         else if (key == "api-version")
             apiVersions << value;
         else if (key == "header-file")
@@ -224,6 +228,8 @@ static bool processProjectFile(QFile& projectFile, QMap<QString, QString>& args)
         args.insert(typesystemPathOption(), typesystemPaths.join(QLatin1String(PATH_SPLITTER)));
     if (!apiVersions.isEmpty())
         args.insert(QLatin1String("api-version"), apiVersions.join(QLatin1Char('|')));
+    if (!languageLevel.isEmpty())
+        args.insert(languageLevelOption(), languageLevel);
     return true;
 }
 
@@ -311,6 +317,8 @@ static void getCommandLineArg(QString arg, int &argNum, QMap<QString, QString> &
             addPathOptionValue(typesystemPathOption(), arg.mid(1), args);
         else if (arg == QLatin1String("h"))
             args.insert(helpOption(), QString());
+        else if (arg.startsWith(QLatin1String("std=")))
+            args.insert(languageLevelOption(), arg.mid(4));
         else
             args.insert(arg, QString());
         return;
@@ -348,6 +356,13 @@ static inline Generators shibokenGenerators()
     return result;
 }
 
+static inline QString languageLevelDescription()
+{
+    return QLatin1String("C++ Language level (c++11..c++17, default=")
+        + QLatin1String(clang::languageLevelOption(clang::emulatedCompilerLanguageLevel()))
+        + QLatin1Char(')');
+}
+
 void printUsage()
 {
     QTextStream s(stdout);
@@ -379,6 +394,8 @@ void printUsage()
         << qMakePair(QLatin1String("-I") + pathSyntax, QString())
         << qMakePair(QLatin1String("include-paths=") + pathSyntax,
                      QLatin1String("Include paths used by the C++ parser"))
+        << qMakePair(languageLevelOption() + QLatin1String("=, -std=<level>"),
+                     languageLevelDescription())
         << qMakePair(QLatin1String("license-file=<license-file>"),
                      QLatin1String("File used for copyright headers of generated files"))
         << qMakePair(QLatin1String("no-suppress-warnings"),
@@ -588,6 +605,18 @@ int main(int argc, char *argv[])
             argsHandler.removeArg(od.first);
     }
 
+    const QString languageLevel = argsHandler.removeArg(languageLevelOption());
+    if (!languageLevel.isEmpty()) {
+        const QByteArray languageLevelBA = languageLevel.toLatin1();
+        const LanguageLevel level = clang::languageLevelFromOption(languageLevelBA.constData());
+        if (level == LanguageLevel::Default) {
+            std::cout << "Invalid argument for language level: \""
+                << languageLevelBA.constData() << "\"\n" << helpHint;
+            return EXIT_FAILURE;
+        }
+        extractor.setLanguageLevel(level);
+    }
+
     if (!argsHandler.noArgs()) {
         errorPrint(argsHandler.errorMessage());
         std::cout << helpHint;
@@ -601,6 +630,7 @@ int main(int argc, char *argv[])
 
     extractor.setCppFileName(cppFileNameFi.absoluteFilePath());
     extractor.setTypeSystem(typeSystemFileName);
+
     if (!extractor.run()) {
         errorPrint(QLatin1String("Error running ApiExtractor."));
         return EXIT_FAILURE;
diff --git a/sources/shiboken2/tests/dumpcodemodel/main.cpp b/sources/shiboken2/tests/dumpcodemodel/main.cpp
index 997e13511..e132c97b3 100644
--- a/sources/shiboken2/tests/dumpcodemodel/main.cpp
+++ b/sources/shiboken2/tests/dumpcodemodel/main.cpp
@@ -28,6 +28,7 @@
 
 #include <abstractmetabuilder_p.h>
 #include <parser/codemodel.h>
+#include <clangparser/compilersupport.h>
 
 #include <QtCore/QCoreApplication>
 #include <QtCore/QCommandLineOption>
@@ -40,6 +41,13 @@
 #include <algorithm>
 #include <iterator>
 
+static inline QString languageLevelDescription()
+{
+    return QLatin1String("C++ Language level (c++11..c++17, default=")
+        + QLatin1String(clang::languageLevelOption(clang::emulatedCompilerLanguageLevel()))
+        + QLatin1Char(')');
+}
+
 int main(int argc, char **argv)
 {
     QCoreApplication app(argc, argv);
@@ -52,6 +60,10 @@ int main(int argc, char **argv)
     QCommandLineOption verboseOption(QStringLiteral("d"),
                                      QStringLiteral("Display verbose output about types"));
     parser.addOption(verboseOption);
+    QCommandLineOption languageLevelOption(QStringLiteral("std"),
+                                           languageLevelDescription(),
+                                           QStringLiteral("level"));
+    parser.addOption(languageLevelOption);
     parser.addPositionalArgument(QStringLiteral("file"), QStringLiteral("C++ source file"));
 
     parser.process(app);
@@ -62,7 +74,19 @@ int main(int argc, char **argv)
     QByteArrayList arguments;
     std::transform(positionalArguments.cbegin(), positionalArguments.cend(),
                    std::back_inserter(arguments), QFile::encodeName);
-    const FileModelItem dom = AbstractMetaBuilderPrivate::buildDom(arguments, 0);
+
+    LanguageLevel level = LanguageLevel::Default;
+    if (parser.isSet(languageLevelOption)) {
+        const QByteArray value = parser.value(languageLevelOption).toLatin1();
+        level = clang::languageLevelFromOption(value.constData());
+        if (level == LanguageLevel::Default) {
+            std::cerr << "Invalid value \"" << value.constData()
+                << "\" for language level option.\n";
+            return -2;
+        }
+    }
+
+    const FileModelItem dom = AbstractMetaBuilderPrivate::buildDom(arguments, level, 0);
     if (dom.isNull()) {
         QString message = QLatin1String("Unable to parse ") + positionalArguments.join(QLatin1Char(' '));
         std::cerr << qPrintable(message) << '\n';