CMake: pro2cmake.py: Better parsing of scopes with else

Parse conditions more exactly as before, enabling proper handling
of else scopes.

Change-Id: Icb5dcc73010be4833b2d1cbc1396191992df1ee4
Reviewed-by: Albert Astals Cid <albert.astals.cid@kdab.com>

7 files changed, 237 insertions, 61 deletions

diff --git a/util/cmake/pro2cmake.py b/util/cmake/pro2cmake.py
index 5537ba38dc..63938d75c8 100755
--- a/util/cmake/pro2cmake.py
+++ b/util/cmake/pro2cmake.py
@@ -32,6 +32,7 @@ from __future__ import annotations
 
 from argparse import ArgumentParser
 import copy
+from itertools import chain
 import os.path
 import re
 import io
@@ -509,10 +510,12 @@ class QmakeParser:
         # Define grammar:
         pp.ParserElement.setDefaultWhitespaceChars(' \t')
 
-        LC = pp.Suppress(pp.Literal('\\') + pp.LineEnd())
-        EOL = pp.Suppress(pp.Optional(pp.pythonStyleComment()) + pp.LineEnd())
-
+        LC = pp.Suppress(pp.Literal('\\\n'))
+        EOL = pp.Suppress(pp.Literal('\n'))
+        Else = pp.Keyword('else')
+        DefineTest = pp.Keyword('defineTest')
         Identifier = pp.Word(pp.alphas + '_', bodyChars=pp.alphanums+'_-./')
+
         Substitution \
             = pp.Combine(pp.Literal('$')
                          + (((pp.Literal('$') + Identifier
@@ -525,32 +528,31 @@ class QmakeParser:
                              | (pp.Literal('$') + pp.Literal('[') + Identifier
                                 + pp.Literal(']'))
                              )))
-        # Do not match word ending in '\' since that breaks line
-        # continuation:-/
         LiteralValuePart = pp.Word(pp.printables, excludeChars='$#{}()')
         SubstitutionValue \
             = pp.Combine(pp.OneOrMore(Substitution | LiteralValuePart
                                       | pp.Literal('$')))
-        Value = (pp.QuotedString(quoteChar='"', escChar='\\')
-                 | SubstitutionValue)
+        Value = pp.NotAny(Else | pp.Literal('}') | EOL | pp.Literal('\\')) \
+                + (pp.QuotedString(quoteChar='"', escChar='\\')
+                     | SubstitutionValue)
 
-        Values = pp.ZeroOrMore(Value)('value')
+        Values = pp.ZeroOrMore(Value + pp.Optional(LC))('value')
 
         Op = pp.Literal('=') | pp.Literal('-=') | pp.Literal('+=') \
             | pp.Literal('*=')
 
-        Operation = Identifier('key') + Op('operation') + Values('value')
-        Load = pp.Keyword('load') + pp.Suppress('(') \
-            + Identifier('loaded') + pp.Suppress(')')
-        Include = pp.Keyword('include') + pp.Suppress('(') \
-            + pp.CharsNotIn(':{=}#)\n')('included') + pp.Suppress(')')
-        Option = pp.Keyword('option') + pp.Suppress('(') \
-            + Identifier('option') + pp.Suppress(')')
-        DefineTest = pp.Suppress(pp.Keyword('defineTest')
-                                 + pp.Suppress('(') + Identifier
-                                 + pp.Suppress(')')
-                                 + pp.nestedExpr(opener='{', closer='}')
-                                 + pp.LineEnd())  # ignore the whole thing...
+        Key = Identifier
+
+        Operation = Key('key') + pp.Optional(LC) \
+            + Op('operation') + pp.Optional(LC) \
+            + Values('value')
+        CallArgs = pp.nestedExpr()
+        CallArgs.setParseAction(lambda x: ' '.join(chain(*x)))
+        Load = pp.Keyword('load') + CallArgs('loaded')
+        Include = pp.Keyword('include') + CallArgs('included')
+        Option = pp.Keyword('option') + CallArgs('option')
+        DefineTestDefinition = pp.Suppress(DefineTest + CallArgs \
+             + pp.nestedExpr(opener='{', closer='}'))  # ignore the whole thing...
         ForLoop = pp.Suppress(pp.Keyword('for') + pp.nestedExpr()
                               + pp.nestedExpr(opener='{', closer='}',
                                               ignoreExpr=None)
@@ -559,45 +561,54 @@ class QmakeParser:
 
         Scope = pp.Forward()
 
-        Statement = pp.Group(Load | Include | Option | DefineTest
-                             | ForLoop | FunctionCall | Operation)
-        StatementLine = Statement + EOL
-        StatementGroup = pp.ZeroOrMore(StatementLine | Scope | EOL)
-
-        Block = pp.Suppress('{') + pp.Optional(EOL) \
-            + pp.ZeroOrMore(EOL | Statement + EOL | Scope) \
-            + pp.Optional(Statement) + pp.Optional(EOL) \
-            + pp.Suppress('}') + pp.Optional(EOL)
-
-        Condition = pp.Optional(pp.White()) + pp.CharsNotIn(':{=}#\\\n')
-        Condition.setParseAction(lambda x: ' '.join(x).strip())
-
-        SingleLineScope = pp.Suppress(pp.Literal(':')) \
-            + pp.Group(Scope | Block | StatementLine)('statements')
-        MultiLineScope = Block('statements')
-
-        SingleLineElse = pp.Suppress(pp.Literal(':')) \
-            + pp.Group(Scope | StatementLine)('else_statements')
-        MultiLineElse = pp.Group(Block)('else_statements')
-        Else = pp.Suppress(pp.Keyword('else')) \
-            + (SingleLineElse | MultiLineElse)
-        Scope <<= pp.Group(Condition('condition')
-                           + (SingleLineScope | MultiLineScope)
-                           + pp.Optional(Else))
+        Statement = pp.Group(Load | Include | Option | ForLoop \
+            | DefineTestDefinition | FunctionCall | Operation)
+        StatementLine = Statement + (EOL | pp.FollowedBy('}'))
+        StatementGroup = pp.ZeroOrMore(StatementLine | Scope | pp.Suppress(EOL))
+
+        Block = pp.Suppress('{')  + pp.Optional(LC | EOL) \
+            + StatementGroup + pp.Optional(LC | EOL) \
+            + pp.Suppress('}') + pp.Optional(LC | EOL)
+
+        ConditionEnd = pp.FollowedBy((pp.Optional(LC) + (pp.Literal(':') \
+                                                         | pp.Literal('{') \
+                                                         | pp.Literal('|'))))
+        ConditionPart = pp.CharsNotIn('#{}|:=\\\n') + pp.Optional(LC) + ConditionEnd
+        Condition = pp.Combine(ConditionPart \
+            + pp.ZeroOrMore((pp.Literal('|') ^ pp.Literal(':')) \
+            + ConditionPart))
+        Condition.setParseAction(lambda x: ' '.join(x).strip().replace(':', ' && ').strip(' && '))
+
+        SingleLineScope = pp.Suppress(pp.Literal(':')) + pp.Optional(LC) \
+            + pp.Group(Block | (Statement + EOL))('statements')
+        MultiLineScope = pp.Optional(LC) + Block('statements')
+
+        SingleLineElse = pp.Suppress(pp.Literal(':')) + pp.Optional(LC) \
+            + (Scope | Block | (Statement + pp.Optional(EOL)))
+        MultiLineElse = Block
+        ElseBranch = pp.Suppress(Else) + (SingleLineElse | MultiLineElse)
+        Scope <<= pp.Optional(LC) \
+            + pp.Group(Condition('condition') \
+            + (SingleLineScope | MultiLineScope) \
+            + pp.Optional(ElseBranch)('else_statements'))
 
         if debug:
-            for ename in 'EOL Identifier Substitution SubstitutionValue ' \
-                         'LiteralValuePart Value Values SingleLineScope ' \
-                         'MultiLineScope Scope SingleLineElse ' \
-                         'MultiLineElse Else Condition Block ' \
-                         'StatementGroup Statement Load Include Option ' \
-                         'DefineTest ForLoop FunctionCall Operation'.split():
+            for ename in 'LC EOL ' \
+                         'Condition ConditionPart ConditionEnd ' \
+                         'Else ElseBranch SingleLineElse MultiLineElse ' \
+                         'SingleLineScope MultiLineScope ' \
+                         'Identifier ' \
+                         'Key Op Values Value ' \
+                         'Scope Block ' \
+                         'StatementGroup StatementLine Statement '\
+                         'Load Include Option DefineTest ForLoop ' \
+                         'FunctionCall CallArgs Operation'.split():
                 expr = locals()[ename]
                 expr.setName(ename)
                 expr.setDebug()
 
         Grammar = StatementGroup('statements')
-        Grammar.ignore(LC)
+        Grammar.ignore(pp.pythonStyleComment())
 
         return Grammar
 
@@ -971,8 +982,8 @@ def simplify_condition(condition: str) -> str:
         condition = condition.replace(' NOT ', ' ~ ')
         condition = condition.replace(' AND ', ' & ')
         condition = condition.replace(' OR ', ' | ')
-        condition = condition.replace(' ON ', 'true')
-        condition = condition.replace(' OFF ', 'false')
+        condition = condition.replace(' ON ', ' true ')
+        condition = condition.replace(' OFF ', ' false ')
 
     try:
         # Generate and simplify condition using sympy:
@@ -989,9 +1000,7 @@ def simplify_condition(condition: str) -> str:
         # sympy did not like our input, so leave this condition alone:
         condition = input_condition
 
-    if condition == '':
-        condition = 'ON'
-    return condition
+    return condition or 'ON'
 
 
 def recursive_evaluate_scope(scope: Scope, parent_condition: str = '',
diff --git a/util/cmake/tests/data/comment_scope.pro b/util/cmake/tests/data/comment_scope.pro
new file mode 100644
index 0000000000..be43cad37d
--- /dev/null
+++ b/util/cmake/tests/data/comment_scope.pro
@@ -0,0 +1,6 @@
+# QtCore can't be compiled with -Wl,-no-undefined because it uses the "environ"
+# variable and on FreeBSD and OpenBSD, this variable is in the final executable itself.
+# OpenBSD 6.0 will include environ in libc.
+freebsd|openbsd: QMAKE_LFLAGS_NOUNDEF =
+
+include(animation/animation.pri)
diff --git a/util/cmake/tests/data/contains_scope.pro b/util/cmake/tests/data/contains_scope.pro
new file mode 100644
index 0000000000..0f51350a45
--- /dev/null
+++ b/util/cmake/tests/data/contains_scope.pro
@@ -0,0 +1,4 @@
+contains(DEFINES,QT_EVAL):include(eval.pri)
+
+HOST_BINS = $$[QT_HOST_BINS]
+
diff --git a/util/cmake/tests/data/multiline_assign.pro b/util/cmake/tests/data/multiline_assign.pro
new file mode 100644
index 0000000000..42a3d0a674
--- /dev/null
+++ b/util/cmake/tests/data/multiline_assign.pro
@@ -0,0 +1,4 @@
+A = 42 \
+  43 \
+  44
+B=23
diff --git a/util/cmake/tests/data/standardpaths.pro b/util/cmake/tests/data/standardpaths.pro
new file mode 100644
index 0000000000..4b45788e4f
--- /dev/null
+++ b/util/cmake/tests/data/standardpaths.pro
@@ -0,0 +1,17 @@
+win32 {
+    !winrt {
+        SOURCES +=io/qstandardpaths_win.cpp 
+    } else {
+        SOURCES +=io/qstandardpaths_winrt.cpp
+    }
+} else:unix {
+    mac {
+        OBJECTIVE_SOURCES += io/qstandardpaths_mac.mm
+    } else:android:!android-embedded {
+        SOURCES += io/qstandardpaths_android.cpp
+    } else:haiku {
+        SOURCES += io/qstandardpaths_haiku.cpp
+    } else {
+        SOURCES += io/qstandardpaths_unix.cpp
+    }
+}
diff --git a/util/cmake/tests/test_parsing.py b/util/cmake/tests/test_parsing.py
index 0802fe4742..e4f9680f60 100755
--- a/util/cmake/tests/test_parsing.py
+++ b/util/cmake/tests/test_parsing.py
@@ -37,7 +37,7 @@ _tests_path = os.path.dirname(os.path.abspath(__file__))
 def validate_op(key, op, value, to_validate):
     assert key == to_validate['key']
     assert op == to_validate['operation']
-    assert value == to_validate['value']
+    assert value == to_validate.get('value', None)
 
 
 def validate_single_op(key, op, value, to_validate):
@@ -71,10 +71,21 @@ def validate_default_else_test(file_name):
 
 def parse_file(file):
     p = QmakeParser(debug=True)
-    result = p.parseFile(file).asDict()
-    assert len(result) == 1
+    result = p.parseFile(file)
+
+    print('\n\n#### Parser result:')
+    print(result)
+    print('\n#### End of parser result.\n')
+
+    print('\n\n####Parser result dictionary:')
+    print(result.asDict())
+    print('\n#### End of parser result dictionary.\n')
+
+    result_dictionary = result.asDict()
+
+    assert len(result_dictionary) == 1
 
-    return result['statements']
+    return result_dictionary['statements']
 
 
 def test_else():
@@ -129,6 +140,13 @@ def test_else8():
     validate_default_else_test(_tests_path + '/data/else8.pro')
 
 
+def test_multiline_assign():
+    result = parse_file(_tests_path + '/data/multiline_assign.pro')
+    assert len(result) == 2
+    validate_op('A', '=', ['42', '43', '44'], result[0])
+    validate_op('B', '=', ['23'], result[1])
+
+
 def test_include():
     result = parse_file(_tests_path + '/data/include.pro')
     assert len(result) == 3
@@ -174,3 +192,65 @@ def test_complex_values():
 def test_function_if():
     result = parse_file(_tests_path + '/data/function_if.pro')
     assert len(result) == 1
+
+
+def test_realworld_standardpaths():
+    result = parse_file(_tests_path + '/data/standardpaths.pro')
+
+    (cond, if_branch, else_branch) = evaluate_condition(result[0])
+    assert cond == 'win32'
+    assert len(if_branch) == 1
+    assert len(else_branch) == 1
+
+    # win32:
+    (cond1, if_branch1, else_branch1) = evaluate_condition(if_branch[0])
+    assert cond1 == '!winrt'
+    assert len(if_branch1) == 1
+    validate_op('SOURCES', '+=', ['io/qstandardpaths_win.cpp'], if_branch1[0])
+    assert len(else_branch1) == 1
+    validate_op('SOURCES', '+=', ['io/qstandardpaths_winrt.cpp'], else_branch1[0])
+
+    # unix:
+    (cond2, if_branch2, else_branch2) = evaluate_condition(else_branch[0])
+    assert cond2 == 'unix'
+    assert len(if_branch2) == 1
+    assert len(else_branch2) == 0
+
+    # mac / else:
+    (cond3, if_branch3, else_branch3) = evaluate_condition(if_branch2[0])
+    assert cond3 == 'mac'
+    assert len(if_branch3) == 1
+    validate_op('OBJECTIVE_SOURCES', '+=', ['io/qstandardpaths_mac.mm'], if_branch3[0])
+    assert len(else_branch3) == 1
+
+    # android / else:
+    (cond4, if_branch4, else_branch4) = evaluate_condition(else_branch3[0])
+    assert cond4 == 'android && !android-embedded'
+    assert len(if_branch4) == 1
+    validate_op('SOURCES', '+=', ['io/qstandardpaths_android.cpp'], if_branch4[0])
+    assert len(else_branch4) == 1
+
+    # haiku / else:
+    (cond5, if_branch5, else_branch5) = evaluate_condition(else_branch4[0])
+    assert cond5 == 'haiku'
+    assert len(if_branch5) == 1
+    validate_op('SOURCES', '+=', ['io/qstandardpaths_haiku.cpp'], if_branch5[0])
+    assert len(else_branch5) == 1
+    validate_op('SOURCES', '+=', ['io/qstandardpaths_unix.cpp'], else_branch5[0])
+
+
+def test_realworld_comment_scope():
+    result = parse_file(_tests_path + '/data/comment_scope.pro')
+    assert len(result) == 2
+    (cond, if_branch, else_branch) = evaluate_condition(result[0])
+    assert cond == 'freebsd|openbsd'
+    assert len(if_branch) == 1
+    validate_op('QMAKE_LFLAGS_NOUNDEF', '=', None, if_branch[0])
+
+    assert result[1].get('included', '') == 'animation/animation.pri'
+
+
+def test_realworld_contains_scope():
+    result = parse_file(_tests_path + '/data/contains_scope.pro')
+    assert len(result) == 2
+
diff --git a/util/cmake/tests/test_scope_handling.py b/util/cmake/tests/test_scope_handling.py
index 2d4bc183d7..1c3406bac8 100755
--- a/util/cmake/tests/test_scope_handling.py
+++ b/util/cmake/tests/test_scope_handling.py
@@ -280,3 +280,59 @@ def test_merge_parent_child_scopes_with_on_child_condition():
     assert r0.getString('test1') == 'parent'
     assert r0.getString('test2') == 'child'
 
+
+# Real world examples:
+
+# qstandardpaths selection:
+
+def test_qstandardpaths_scopes():
+    # top level:
+    scope1 = _new_scope(condition='ON', scope_id=1)
+
+    # win32 {
+    scope2 = _new_scope(parent_scope=scope1, condition='WIN32')
+    #     !winrt {
+    #         SOURCES += io/qstandardpaths_win.cpp
+    scope3 = _new_scope(parent_scope=scope2, condition='NOT WINRT',
+                        SOURCES='qsp_win.cpp')
+    #     } else {
+    #         SOURCES += io/qstandardpaths_winrt.cpp
+    scope4 = _new_scope(parent_scope=scope2, condition='else',
+                        SOURCES='qsp_winrt.cpp')
+    #     }
+    # else: unix {
+    scope5 = _new_scope(parent_scope=scope1, condition='else')
+    scope6 = _new_scope(parent_scope=scope5, condition='UNIX')
+    #     mac {
+    #         OBJECTIVE_SOURCES += io/qstandardpaths_mac.mm
+    scope7 = _new_scope(parent_scope=scope6, condition='APPLE_OSX', SOURCES='qsp_mac.mm')
+    #     } else:android:!android-embedded {
+    #         SOURCES += io/qstandardpaths_android.cpp
+    scope8 = _new_scope(parent_scope=scope6, condition='else')
+    scope9 = _new_scope(parent_scope=scope8,
+                        condition='ANDROID AND NOT ANDROID_EMBEDDED',
+                        SOURCES='qsp_android.cpp')
+    #     } else:haiku {
+    #              SOURCES += io/qstandardpaths_haiku.cpp
+    scope10 = _new_scope(parent_scope=scope8, condition='else')
+    scope11 = _new_scope(parent_scope=scope10, condition='HAIKU', SOURCES='qsp_haiku.cpp')
+    #     } else {
+    #         SOURCES +=io/qstandardpaths_unix.cpp
+    scope12 = _new_scope(parent_scope=scope10, condition='else', SOURCES='qsp_unix.cpp')
+    #     }
+    # }
+
+    recursive_evaluate_scope(scope1)
+
+    assert scope1.total_condition == 'ON'
+    assert scope2.total_condition == 'WIN32'
+    assert scope3.total_condition == 'WIN32 AND NOT WINRT'
+    assert scope4.total_condition == 'WINRT'
+    assert scope5.total_condition == 'UNIX'
+    assert scope6.total_condition == 'UNIX'
+    assert scope7.total_condition == 'APPLE_OSX'
+    assert scope8.total_condition == 'UNIX AND NOT APPLE_OSX'
+    assert scope9.total_condition == 'ANDROID AND NOT ANDROID_EMBEDDED AND NOT APPLE_OSX'
+    assert scope10.total_condition == 'UNIX AND NOT APPLE_OSX AND (ANDROID_EMBEDDED OR NOT ANDROID)'
+    assert scope11.total_condition == 'HAIKU AND UNIX AND NOT APPLE_OSX AND (ANDROID_EMBEDDED OR NOT ANDROID)'
+    assert scope12.total_condition == 'UNIX AND NOT APPLE_OSX AND NOT HAIKU AND (ANDROID_EMBEDDED OR NOT ANDROID)'