qmlformat: Fix some language features being unsupported

qmlformat now supports:
- arrow functions
- generator functions
- this expressions
- object patterns
- regex literals
- type expressions
- plain expressions

Aborts if an error occurs during dumping now.

Also now automatically tests qmlformat against all example / test qml
files.

Change-Id: Idc24004c6f2c1cd65289bcad75985a1ef047c8d2
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>

6 files changed, 232 insertions, 20 deletions

diff --git a/tests/auto/qml/qml.pro b/tests/auto/qml/qml.pro
index 1644632ec6..7a08cc805c 100644
--- a/tests/auto/qml/qml.pro
+++ b/tests/auto/qml/qml.pro
@@ -12,7 +12,6 @@ PUBLICTESTS += \
     qqmlfileselector
 
 PUBLICTESTS += \
-    qmlformat \
     qmlmin \
     qqmlcomponent \
     qqmlconsole \
@@ -92,7 +91,7 @@ SUBDIRS += $$METATYPETESTS
 qtConfig(process) {
     qtConfig(qml-debug): SUBDIRS += debugger
     !boot2qt {
-        SUBDIRS += qmllint qmlplugindump
+        SUBDIRS += qmlformat qmllint qmlplugindump
     }
 }
 
diff --git a/tests/auto/qml/qmlformat/qmlformat.pro b/tests/auto/qml/qmlformat/qmlformat.pro
index 9f8a44bc09..a6ae391711 100644
--- a/tests/auto/qml/qmlformat/qmlformat.pro
+++ b/tests/auto/qml/qmlformat/qmlformat.pro
@@ -3,6 +3,7 @@ TARGET = tst_qmlformat
 macos:CONFIG -= app_bundle
 
 SOURCES += tst_qmlformat.cpp
+DEFINES += SRCDIR=\\\"$$PWD\\\"
 
 include (../../shared/util.pri)
 
diff --git a/tests/auto/qml/qmlformat/tst_qmlformat.cpp b/tests/auto/qml/qmlformat/tst_qmlformat.cpp
index 7ad9c99d83..95c8e88f21 100644
--- a/tests/auto/qml/qmlformat/tst_qmlformat.cpp
+++ b/tests/auto/qml/qmlformat/tst_qmlformat.cpp
@@ -42,11 +42,22 @@ private Q_SLOTS:
     void testFormat();
     void testFormatNoSort();
 
+
+#if !defined(QTEST_CROSS_COMPILED) // sources not available when cross compiled
+    void testExample();
+    void testExample_data();
+#endif
+
 private:
     QString readTestFile(const QString &path);
     QString runQmlformat(const QString &fileToFormat, bool sortImports, bool shouldSucceed);
 
     QString m_qmlformatPath;
+    QStringList m_excludedDirs;
+    QStringList m_invalidFiles;
+
+    QStringList findFiles(const QDir &);
+    bool isInvalidFile(const QFileInfo &fileName) const;
 };
 
 void TestQmlformat::initTestCase()
@@ -60,6 +71,91 @@ void TestQmlformat::initTestCase()
         QString message = QStringLiteral("qmlformat executable not found (looked for %0)").arg(m_qmlformatPath);
         QFAIL(qPrintable(message));
     }
+
+    // Add directories you want excluded here
+
+    // These snippets are not expected to run on their own.
+    m_excludedDirs << "doc/src/snippets/qml/visualdatamodel_rootindex";
+    m_excludedDirs << "doc/src/snippets/qml/qtbinding";
+    m_excludedDirs << "doc/src/snippets/qml/imports";
+    m_excludedDirs << "doc/src/snippets/qtquick1/visualdatamodel_rootindex";
+    m_excludedDirs << "doc/src/snippets/qtquick1/qtbinding";
+    m_excludedDirs << "doc/src/snippets/qtquick1/imports";
+    m_excludedDirs << "tests/manual/v4";
+    m_excludedDirs << "tests/auto/qml/ecmascripttests";
+    m_excludedDirs << "tests/auto/qml/qmllint";
+
+    // Add invalid files (i.e. files with syntax errors)
+    m_invalidFiles << "tests/auto/quick/qquickloader/data/InvalidSourceComponent.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/signal.2.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/signal.3.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/signal.5.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/property.4.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/empty.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/missingObject.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/insertedSemicolon.1.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/nonexistantProperty.5.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/invalidRoot.1.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/invalidQmlEnumValue.1.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/invalidQmlEnumValue.2.qml";
+    m_invalidFiles << "tests/auto/qml/qquickfolderlistmodel/data/dummy.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/stringParsing_error.1.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/stringParsing_error.2.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/stringParsing_error.3.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/stringParsing_error.4.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/stringParsing_error.5.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/stringParsing_error.6.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/numberParsing_error.1.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/numberParsing_error.2.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlecmascript/data/incrDecrSemicolon_error1.qml";
+    m_invalidFiles << "tests/auto/qml/debugger/qqmlpreview/data/broken.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/fuzzed.2.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/fuzzed.3.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/requiredProperties.2.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/requiredProperties.3.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/nullishCoalescing_LHS_And.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/nullishCoalescing_LHS_And.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/nullishCoalescing_LHS_Or.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/nullishCoalescing_RHS_And.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/nullishCoalescing_RHS_Or.qml";
+    m_invalidFiles << "tests/auto/qml/qqmllanguage/data/typeAnnotations.2.qml";
+    m_invalidFiles << "tests/auto/qml/qqmlparser/data/disallowedtypeannotations/qmlnestedfunction.qml";
+}
+
+QStringList TestQmlformat::findFiles(const QDir &d)
+{
+    for (int ii = 0; ii < m_excludedDirs.count(); ++ii) {
+        QString s = m_excludedDirs.at(ii);
+        if (d.absolutePath().endsWith(s))
+            return QStringList();
+    }
+
+    QStringList rv;
+
+    QStringList files = d.entryList(QStringList() << QLatin1String("*.qml"),
+                                    QDir::Files);
+    foreach (const QString &file, files) {
+        rv << d.absoluteFilePath(file);
+    }
+
+    QStringList dirs = d.entryList(QDir::Dirs | QDir::NoDotAndDotDot |
+                                   QDir::NoSymLinks);
+    foreach (const QString &dir, dirs) {
+        QDir sub = d;
+        sub.cd(dir);
+        rv << findFiles(sub);
+    }
+
+    return rv;
+}
+
+bool TestQmlformat::isInvalidFile(const QFileInfo &fileName) const
+{
+    for (const QString &invalidFile : m_invalidFiles) {
+        if (fileName.absoluteFilePath().endsWith(invalidFile))
+            return true;
+    }
+    return false;
 }
 
 QString TestQmlformat::readTestFile(const QString &path)
@@ -74,18 +170,46 @@ QString TestQmlformat::readTestFile(const QString &path)
 
 void TestQmlformat::testFormat()
 {
-    QCOMPARE(runQmlformat("Example1.qml", true, true), readTestFile("Example1.formatted.qml"));
+    QCOMPARE(runQmlformat(testFile("Example1.qml"), true, true), readTestFile("Example1.formatted.qml"));
 }
 
 void TestQmlformat::testFormatNoSort()
 {
-    QCOMPARE(runQmlformat("Example1.qml", false, true), readTestFile("Example1.formatted.nosort.qml"));
+    QCOMPARE(runQmlformat(testFile("Example1.qml"), false, true), readTestFile("Example1.formatted.nosort.qml"));
 }
 
+#if !defined(QTEST_CROSS_COMPILED) // sources not available when cross compiled
+void TestQmlformat::testExample_data()
+{
+    QTest::addColumn<QString>("file");
+
+    QString examples = QLatin1String(SRCDIR) + "/../../../../examples/";
+    QString tests = QLatin1String(SRCDIR) + "/../../../../tests/";
+
+    QStringList files;
+    files << findFiles(QDir(examples));
+    files << findFiles(QDir(tests));
+
+    for (const QString &file : files)
+        QTest::newRow(qPrintable(file)) << file;
+}
+#endif
+
+#if !defined(QTEST_CROSS_COMPILED) // sources not available when cross compiled
+void TestQmlformat::testExample()
+{
+    QFETCH(QString, file);
+    QString output = runQmlformat(file, true, !isInvalidFile(file));
+
+    if (!isInvalidFile(file))
+        QVERIFY(!output.isEmpty());
+}
+#endif
+
 QString TestQmlformat::runQmlformat(const QString &fileToFormat, bool sortImports, bool shouldSucceed)
 {
     QStringList args;
-    args << testFile(fileToFormat);
+    args << fileToFormat;
 
     if (!sortImports)
         args << "-n";
diff --git a/tools/qmlformat/dumpastvisitor.cpp b/tools/qmlformat/dumpastvisitor.cpp
index 716be0a9a7..20ebef8927 100644
--- a/tools/qmlformat/dumpastvisitor.cpp
+++ b/tools/qmlformat/dumpastvisitor.cpp
@@ -28,6 +28,8 @@
 
 #include "dumpastvisitor.h"
 
+#include <QtQml/private/qqmljslexer_p.h>
+
 DumpAstVisitor::DumpAstVisitor(Node *rootNode, CommentAstVisitor *comment): m_comment(comment)
 {
     // Add all completely orphaned comments
@@ -257,6 +259,22 @@ QString DumpAstVisitor::parseFormalParameterList(FormalParameterList *list)
     return result;
 }
 
+QString DumpAstVisitor::parsePatternProperty(PatternProperty *property)
+{
+    return escapeString(property->name->asString())+": "+parsePatternElement(property, false);
+}
+
+QString DumpAstVisitor::parsePatternPropertyList(PatternPropertyList *list)
+{
+    QString result = "";
+
+    for (auto *item = list; item != nullptr; item = item->next) {
+        result += formatLine(parsePatternProperty(item->property) + (item->next != nullptr ? "," : ""));
+    }
+
+    return result;
+}
+
 QString DumpAstVisitor::parseExpression(ExpressionNode *expression)
 {
     if (expression == nullptr)
@@ -288,13 +306,23 @@ QString DumpAstVisitor::parseExpression(ExpressionNode *expression)
         auto *functExpr = cast<FunctionExpression *>(expression);
 
         m_indentLevel++;
-        QString result = "function";
+        QString result;
+
+        if (!functExpr->isArrowFunction) {
+            result += "function";
 
-        if (!functExpr->name.isEmpty())
-            result += " " + functExpr->name;
+            if (functExpr->isGenerator)
+                result += "*";
 
-        result += "("+parseFormalParameterList(functExpr->formals)+") {\n"
-                + parseStatementList(functExpr->body);
+            if (!functExpr->name.isEmpty())
+                result += " " + functExpr->name;
+
+            result += "("+parseFormalParameterList(functExpr->formals)+") {\n"
+                    + parseStatementList(functExpr->body);
+        } else {
+             result += "("+parseFormalParameterList(functExpr->formals)+") => {\n";
+             result += parseStatementList(functExpr->body);
+        }
 
         m_indentLevel--;
 
@@ -304,6 +332,8 @@ QString DumpAstVisitor::parseExpression(ExpressionNode *expression)
     }
     case Node::Kind_NullExpression:
         return "null";
+    case Node::Kind_ThisExpression:
+        return "this";
     case Node::Kind_PostIncrementExpression:
         return parseExpression(cast<PostIncrementExpression *>(expression)->base)+"++";
     case Node::Kind_PreIncrementExpression:
@@ -371,6 +401,44 @@ QString DumpAstVisitor::parseExpression(ExpressionNode *expression)
 
         return result;
     }
+    case Node::Kind_ObjectPattern: {
+        auto *objectPattern = cast<ObjectPattern*>(expression);
+        QString result = "{\n";
+
+        m_indentLevel++;
+        result += parsePatternPropertyList(objectPattern->properties);
+        m_indentLevel--;
+
+        result += formatLine("}", false);
+
+        return result;
+    }
+    case Node::Kind_Expression: {
+        auto* expr = cast<Expression*>(expression);
+        return parseExpression(expr->left)+", "+parseExpression(expr->right);
+    }
+    case Node::Kind_Type: {
+        auto* type = reinterpret_cast<Type*>(expression);
+
+        return parseUiQualifiedId(type->typeId);
+    }
+    case Node::Kind_RegExpLiteral: {
+        auto* regexpLiteral = cast<RegExpLiteral*>(expression);
+        QString result = "/"+regexpLiteral->pattern+"/";
+
+        if (regexpLiteral->flags & QQmlJS::Lexer::RegExp_Unicode)
+            result += "u";
+        if (regexpLiteral->flags & QQmlJS::Lexer::RegExp_Global)
+            result += "g";
+        if (regexpLiteral->flags & QQmlJS::Lexer::RegExp_Multiline)
+            result += "m";
+        if (regexpLiteral->flags & QQmlJS::Lexer::RegExp_Sticky)
+            result += "y";
+        if (regexpLiteral->flags & QQmlJS::Lexer::RegExp_IgnoreCase)
+            result += "i";
+
+        return result;
+    }
     default:
         m_error = true;
         return "unknown_expression_"+QString::number(expression->kind);
@@ -383,7 +451,7 @@ QString DumpAstVisitor::parseVariableDeclarationList(VariableDeclarationList *li
 
     for (auto *item = list; item != nullptr; item = item->next) {
         result += parsePatternElement(item->declaration, (item == list))
-               + (item->next != nullptr ? ", " : "");
+                + (item->next != nullptr ? ", " : "");
     }
 
     return result;
@@ -469,7 +537,7 @@ QString DumpAstVisitor::parseStatement(Statement *statement, bool blockHasNext,
     case Node::Kind_VariableStatement:
         return parseVariableDeclarationList(cast<VariableStatement *>(statement)->declarations);
     case Node::Kind_ReturnStatement:
-         return "return "+parseExpression(cast<ReturnStatement *>(statement)->expression);
+        return "return "+parseExpression(cast<ReturnStatement *>(statement)->expression);
     case Node::Kind_ContinueStatement:
         return "continue";
     case Node::Kind_BreakStatement:
@@ -705,7 +773,7 @@ bool DumpAstVisitor::visit(UiPublicMember *node) {
 
         addLine("signal "+node->name.toString()+"("+parseUiParameterList(node->parameters) + ")"
                 + commentBackInline);
-    break;
+        break;
     case UiPublicMember::Property: {
         if (m_firstProperty) {
             if (m_firstOfAll)
@@ -912,7 +980,15 @@ bool DumpAstVisitor::visit(FunctionDeclaration *node) {
     addNewLine();
 
     addLine(getComment(node, Comment::Location::Front));
-    addLine("function "+node->name+"("+parseFormalParameterList(node->formals)+") {");
+
+    QString head = "function";
+
+    if (node->isGenerator)
+        head += "*";
+
+    head += " "+node->name+"("+parseFormalParameterList(node->formals)+") {";
+
+    addLine(head);
     m_indentLevel++;
     m_result += parseStatementList(node->body);
     m_indentLevel--;
@@ -970,8 +1046,8 @@ bool DumpAstVisitor::visit(UiImport *node) {
         result += parseUiQualifiedId(node->importUri);
 
     if (node->version) {
-       result += " " + QString::number(node->version->majorVersion) + "."
-               + QString::number(node->version->minorVersion);
+        result += " " + QString::number(node->version->majorVersion) + "."
+                + QString::number(node->version->minorVersion);
     }
 
     if (node->asToken.isValid()) {
diff --git a/tools/qmlformat/dumpastvisitor.h b/tools/qmlformat/dumpastvisitor.h
index e73a7b628f..2001f4366e 100644
--- a/tools/qmlformat/dumpastvisitor.h
+++ b/tools/qmlformat/dumpastvisitor.h
@@ -89,6 +89,9 @@ private:
     QString parsePatternElement(PatternElement *element, bool scope = true);
     QString parsePatternElementList(PatternElementList *element);
 
+    QString parsePatternProperty(PatternProperty *property);
+    QString parsePatternPropertyList(PatternPropertyList *list);
+
     QString parseArgumentList(ArgumentList *list);
 
     QString parseUiParameterList(UiParameterList *list);
diff --git a/tools/qmlformat/main.cpp b/tools/qmlformat/main.cpp
index bca788d316..036fbe9748 100644
--- a/tools/qmlformat/main.cpp
+++ b/tools/qmlformat/main.cpp
@@ -43,7 +43,7 @@
 #include "dumpastvisitor.h"
 #include "restructureastvisitor.h"
 
-bool parseFile(const QString& filename, bool inplace, bool verbose, bool sortImports)
+bool parseFile(const QString& filename, bool inplace, bool verbose, bool sortImports, bool force)
 {
     QFile file(filename);
 
@@ -101,8 +101,14 @@ bool parseFile(const QString& filename, bool inplace, bool verbose, bool sortImp
 
     DumpAstVisitor dump(parser.rootNode(), &comment);
 
-    if (dump.error())
-        qWarning().noquote() << "An error has occurred. The output may not be reliable.";
+    if (dump.error()) {
+        if (force) {
+            qWarning().noquote() << "An error has occurred. The output may not be reliable.";
+        } else {
+            qWarning().noquote() << "Am error has occurred. Aborting.";
+            return false;
+        }
+   }
 
     if (inplace) {
         if (verbose)
@@ -145,6 +151,9 @@ int main(int argc, char *argv[])
     parser.addOption(QCommandLineOption({"i", "inplace"},
                      QStringLiteral("Edit file in-place instead of outputting to stdout.")));
 
+    parser.addOption(QCommandLineOption({"f", "force"},
+                     QStringLiteral("Continue even if an error has occurred.")));
+
     parser.addPositionalArgument("filenames", "files to be processed by qmlformat");
 
     parser.process(app);
@@ -155,7 +164,7 @@ int main(int argc, char *argv[])
         parser.showHelp(-1);
 
     for (const QString& file: parser.positionalArguments()) {
-        if (!parseFile(file, parser.isSet("inplace"), parser.isSet("verbose"), !parser.isSet("no-sort")))
+        if (!parseFile(file, parser.isSet("inplace"), parser.isSet("verbose"), !parser.isSet("no-sort"), parser.isSet("force")))
             success = false;
     }
 #endif