CMake: pro2cmake.py: Merge more scopes

* Remove scopes with condition 'OFF'
* Merge scopes with identical conditions

This e.g. merges children with a condition that simplifies to
'ON' with their parent scope.

Change-Id: Ieb3d60e1234f189ac45869853555ca8c0cfb5c76
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

2 files changed, 328 insertions, 40 deletions

diff --git a/util/cmake/pro2cmake.py b/util/cmake/pro2cmake.py
index 6883a117e0..d4d352d4f6 100755
--- a/util/cmake/pro2cmake.py
+++ b/util/cmake/pro2cmake.py
@@ -27,14 +27,17 @@
 ##
 #############################################################################
 
+
+from __future__ import annotations
+
 from argparse import ArgumentParser
+import copy
 import os.path
 import re
 import io
 import typing
 
 from sympy.logic import (simplify_logic, And, Or, Not,)
-from sympy.core import SympifyError
 import pyparsing as pp
 
 from helper import map_qt_library, map_qt_base_library, featureName, \
@@ -204,9 +207,11 @@ class RemoveOperation(Operation):
 
 
 class Scope:
-    def __init__(self, parent_scope: typing.Optional['Scope'],
+    def __init__(self, *,
+                 parent_scope: typing.Optional[Scope],
                  file: typing.Optional[str] = None, condition: str = '',
-                 base_dir: str = '') -> None:
+                 base_dir: str = '',
+                 operations: typing.Mapping[str, typing.List[Operation]] = {}) -> None:
         if parent_scope:
             parent_scope._add_child(self)
         else:
@@ -223,7 +228,7 @@ class Scope:
         self._file = file
         self._condition = map_condition(condition)
         self._children = []  # type: typing.List[Scope]
-        self._operations = {}  # type: typing.Dict[str, typing.List[Operation]]
+        self._operations = copy.deepcopy(operations)
         self._visited_keys = set()  # type: typing.Set[str]
         self._total_condition = None  # type: typing.Optional[str]
 
@@ -244,6 +249,9 @@ class Scope:
             else:
                 self._operations[key] = other._operations[key]
 
+    def parent(self) -> typing.Optional[Scope]:
+        return self._parent
+
     def basedir(self) -> str:
         return self._basedir
 
@@ -253,7 +261,7 @@ class Scope:
     @staticmethod
     def FromDict(parent_scope: typing.Optional['Scope'],
                  file: str, statements, cond: str = '', base_dir: str = ''):
-        scope = Scope(parent_scope, file, cond, base_dir)
+        scope = Scope(parent_scope=parent_scope, file=file, condition=cond, base_dir=base_dir)
         for statement in statements:
             if isinstance(statement, list):  # Handle skipped parts...
                 assert not statement
@@ -519,13 +527,11 @@ def parseProFile(file: str, *, debug=False):
 
 
 def map_condition(condition: str) -> str:
-    print('##### Mapping condition: {}.'.format(condition))
     re.sub(r'if\s*\((.*?)\)', r'\1', condition)
     re.sub(r'(^|[^a-zA-Z0-9_])isEmpty\s*\((.*?)\)', r'\2_ISEMPTY', condition)
     re.sub(r'(^|[^a-zA-Z0-9_])contains\s*\((.*?), (.*)?\)',
            r'\2___contains___\3', condition)
     re.sub(r'\s*==\s*', '___STREQUAL___', condition)
-    print('   # after regexp: {}.'.format(condition))
 
     condition = condition.replace('*', '_x_')
     condition = condition.replace('.$$', '__ss_')
@@ -860,6 +866,8 @@ def simplify_condition(condition: str) -> str:
         # sympy did not like our input, so leave this condition alone:
         condition = input_condition
 
+    if condition == '':
+        condition = 'ON'
     return condition
 
 
@@ -934,9 +942,8 @@ def write_extend_target(cm_fh: typing.IO[str], target: str,
 
 
 def flatten_scopes(scope: Scope) -> typing.List[Scope]:
-    result = []  # type: typing.List[Scope]
+    result = [scope]  # type: typing.List[Scope]
     for c in scope.children():
-        result.append(c)
         result += flatten_scopes(c)
     return result
 
@@ -944,16 +951,19 @@ def flatten_scopes(scope: Scope) -> typing.List[Scope]:
 def merge_scopes(scopes: typing.List[Scope]) -> typing.List[Scope]:
     result = []  # type: typing.List[Scope]
 
-    current_scope = None
+    # Merge scopes with their parents:
+    known_scopes = {}  # type: typing.Mapping[str, Scope]
     for scope in scopes:
-        if not current_scope \
-                or scope.total_condition() != current_scope.total_condition():
-            if current_scope:
-                result.append(current_scope)
-            current_scope = scope
-            continue
-
-        current_scope.merge(scope)
+        total_condition = scope.total_condition()
+        if total_condition == 'OFF':
+            # ignore this scope entirely!
+            pass
+        elif total_condition in known_scopes:
+            known_scopes[total_condition].merge(scope)
+        else:
+            # Keep everything else:
+            result.append(scope)
+            known_scopes[total_condition] = scope
 
     return result
 
@@ -963,14 +973,28 @@ def write_main_part(cm_fh: typing.IO[str], name: str, typename: str,
                     extra_lines: typing.List[str] = [],
                     indent: int = 0,
                     **kwargs: typing.Any):
+    # Evaluate total condition of all scopes:
+    recursive_evaluate_scope(scope)
+
+    # Get a flat list of all scopes but the main one:
+    scopes = flatten_scopes(scope)
+    total_scopes = len(scopes)
+    # Merge scopes based on their conditions:
+    scopes = merge_scopes(scopes)
+    print("xxxxxx {} scopes, {} after merging!".format(total_scopes, len(scopes)))
+
+    assert len(scopes)
+    assert scopes[0].total_condition() == 'ON'
+
+    # Now write out the scopes:
     write_header(cm_fh, name, typename, indent=indent)
 
     cm_fh.write('{}{}({}\n'.format(spaces(indent), cmake_function, name))
     for extra_line in extra_lines:
         cm_fh.write('{}    {}\n'.format(spaces(indent), extra_line))
 
-    ignored_keys = write_sources_section(cm_fh, scope, indent=indent, **kwargs)
-    ignored_keys_report = write_ignored_keys(scope, ignored_keys,
+    ignored_keys = write_sources_section(cm_fh, scopes[0], indent=indent, **kwargs)
+    ignored_keys_report = write_ignored_keys(scopes[0], ignored_keys,
                                              spaces(indent + 1))
     if ignored_keys_report:
         cm_fh.write(ignored_keys_report)
@@ -979,26 +1003,12 @@ def write_main_part(cm_fh: typing.IO[str], name: str, typename: str,
     cm_fh.write('{})\n'.format(spaces(indent)))
 
     # Scopes:
-    if not scope.children():
+    if len(scopes) == 1:
         return
 
     write_scope_header(cm_fh, indent=indent)
 
-    # Evaluate total condition of all scopes:
-    for c in scope.children():
-        recursive_evaluate_scope(c)
-
-    # Get a flat list of all scopes but the main one:
-    scopes = flatten_scopes(scope)
-
-    scopes = sorted(scopes, key=lambda x: x.total_condition())
-    print("xxxxxx Sorted to {} scopes!".format(len(scopes)))
-
-    # Merge scopes with identical conditions:
-    scopes = merge_scopes(scopes)
-    print("xxxxxx Merged to {} scopes!".format(len(scopes)))
-
-    for c in scopes:
+    for c in scopes[1:]:
         write_extend_target(cm_fh, name, c, indent=indent)
 
 
@@ -1118,10 +1128,6 @@ def do_include(scope: Scope, *, debug: bool = False) -> None:
         include_file = i
         if not include_file:
             continue
-        if '/3rdparty/' in include_file:
-            print('    ****: Ignoring include file in 3rdparty: {}.'
-                  .format(include_file))
-            continue
         if not os.path.isfile(include_file):
             print('    XXXX: Failed to include {}.'.format(include_file))
             continue
diff --git a/util/cmake/tests/test_scope_handling.py b/util/cmake/tests/test_scope_handling.py
new file mode 100755
index 0000000000..3977f2291f
--- /dev/null
+++ b/util/cmake/tests/test_scope_handling.py
@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the plugins of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+from pro2cmake import Scope, SetOperation, merge_scopes, recursive_evaluate_scope
+
+import pytest
+import typing
+
+ScopeList = typing.List[Scope]
+
+def _map_to_operation(**kwargs):
+    result = {}  # type: typing.Mapping[str, typing.List[SetOperation]]
+    for (key, value) in kwargs.items():
+        result[key] = [SetOperation(value)]
+    return result
+
+
+def _new_scope(*, parent_scope=None, condition='', **kwargs) -> Scope:
+    return Scope(parent_scope=parent_scope,
+                 file='file1', condition=condition, operations=_map_to_operation(**kwargs))
+
+
+def _evaluate_scopes(scopes: ScopeList) -> ScopeList:
+    for s in scopes:
+        if not s.parent():
+            recursive_evaluate_scope(s)
+    return scopes
+
+
+def _validate(input_scopes: ScopeList, output_scopes: ScopeList):
+    merged_scopes = merge_scopes(input_scopes)
+    assert merged_scopes == output_scopes
+
+
+def test_evaluate_one_scope():
+    scope = _new_scope(condition='QT_FEATURE_foo', test1='bar')
+
+    input_scope = scope
+    recursive_evaluate_scope(scope)
+    assert scope == input_scope
+
+
+def test_evaluate_child_scope():
+    scope = _new_scope(condition='QT_FEATURE_foo', test1='bar')
+    _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar')
+
+    input_scope = scope
+    recursive_evaluate_scope(scope)
+
+    assert scope.total_condition() == 'QT_FEATURE_foo'
+    assert len(scope.children()) == 1
+    assert scope.getString('test1') == 'bar'
+    assert scope.getString('test2', 'not found') == 'not found'
+
+    child = scope.children()[0]
+    assert child.total_condition() == 'QT_FEATURE_bar AND QT_FEATURE_foo'
+    assert child.getString('test1', 'not found') == 'not found'
+    assert child.getString('test2') == 'bar'
+
+
+def test_evaluate_two_child_scopes():
+    scope = _new_scope(condition='QT_FEATURE_foo', test1='bar')
+    _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar')
+    _new_scope(parent_scope=scope, condition='QT_FEATURE_buz', test3='buz')
+
+    input_scope = scope
+    recursive_evaluate_scope(scope)
+
+    assert scope.total_condition() == 'QT_FEATURE_foo'
+    assert len(scope.children()) == 2
+    assert scope.getString('test1') == 'bar'
+    assert scope.getString('test2', 'not found') == 'not found'
+    assert scope.getString('test3', 'not found') == 'not found'
+
+    child1 = scope.children()[0]
+    assert child1.total_condition() == 'QT_FEATURE_bar AND QT_FEATURE_foo'
+    assert child1.getString('test1', 'not found') == 'not found'
+    assert child1.getString('test2') == 'bar'
+    assert child1.getString('test3', 'not found') == 'not found'
+
+    child2 = scope.children()[1]
+    assert child2.total_condition() == 'QT_FEATURE_buz AND QT_FEATURE_foo'
+    assert child2.getString('test1', 'not found') == 'not found'
+    assert child2.getString('test2') == ''
+    assert child2.getString('test3', 'not found') == 'buz'
+
+
+def test_evaluate_else_child_scopes():
+    scope = _new_scope(condition='QT_FEATURE_foo', test1='bar')
+    _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar')
+    _new_scope(parent_scope=scope, condition='else', test3='buz')
+
+    input_scope = scope
+    recursive_evaluate_scope(scope)
+
+    assert scope.total_condition() == 'QT_FEATURE_foo'
+    assert len(scope.children()) == 2
+    assert scope.getString('test1') == 'bar'
+    assert scope.getString('test2', 'not found') == 'not found'
+    assert scope.getString('test3', 'not found') == 'not found'
+
+    child1 = scope.children()[0]
+    assert child1.total_condition() == 'QT_FEATURE_bar AND QT_FEATURE_foo'
+    assert child1.getString('test1', 'not found') == 'not found'
+    assert child1.getString('test2') == 'bar'
+    assert child1.getString('test3', 'not found') == 'not found'
+
+    child2 = scope.children()[1]
+    assert child2.total_condition() == 'QT_FEATURE_foo AND NOT QT_FEATURE_bar'
+    assert child2.getString('test1', 'not found') == 'not found'
+    assert child2.getString('test2') == ''
+    assert child2.getString('test3', 'not found') == 'buz'
+
+
+def test_evaluate_invalid_else_child_scopes():
+    scope = _new_scope(condition='QT_FEATURE_foo', test1='bar')
+    _new_scope(parent_scope=scope, condition='else', test3='buz')
+    _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar')
+
+    input_scope = scope
+    with pytest.raises(AssertionError):
+        recursive_evaluate_scope(scope)
+
+
+def test_merge_empty_scope_list():
+    _validate([], [])
+
+
+def test_merge_one_scope():
+    scopes = [_new_scope(test='foo')]
+
+    recursive_evaluate_scope(scopes[0])
+
+    _validate(scopes, scopes)
+
+
+def test_merge_one_on_scope():
+    scopes = [_new_scope(condition='ON', test='foo')]
+
+    recursive_evaluate_scope(scopes[0])
+
+    _validate(scopes, scopes)
+
+
+def test_merge_one_off_scope():
+    scopes = [_new_scope(condition='OFF', test='foo')]
+
+    recursive_evaluate_scope(scopes[0])
+
+    _validate(scopes, [])
+
+
+def test_merge_one_conditioned_scope():
+    scopes = [_new_scope(condition='QT_FEATURE_foo', test='foo')]
+
+    recursive_evaluate_scope(scopes[0])
+
+    _validate(scopes, scopes)
+
+
+def test_merge_two_scopes_with_same_condition():
+    scopes = [_new_scope(condition='QT_FEATURE_bar', test='foo'),
+              _new_scope(condition='QT_FEATURE_bar', test2='bar')]
+
+    recursive_evaluate_scope(scopes[0])
+    recursive_evaluate_scope(scopes[1])
+
+    result = merge_scopes(scopes)
+
+    assert len(result) == 1
+    r0 = result[0]
+    assert r0.total_condition() == 'QT_FEATURE_bar'
+    assert r0.getString('test') == 'foo'
+    assert r0.getString('test2') == 'bar'
+
+
+def test_merge_three_scopes_two_with_same_condition():
+    scopes = [_new_scope(condition='QT_FEATURE_bar', test='foo'),
+              _new_scope(condition='QT_FEATURE_baz', test1='buz'),
+              _new_scope(condition='QT_FEATURE_bar', test2='bar')]
+
+    recursive_evaluate_scope(scopes[0])
+    recursive_evaluate_scope(scopes[1])
+    recursive_evaluate_scope(scopes[2])
+
+    result = merge_scopes(scopes)
+
+    assert len(result) == 2
+    r0 = result[0]
+    assert r0.total_condition() == 'QT_FEATURE_bar'
+    assert r0.getString('test') == 'foo'
+    assert r0.getString('test2') == 'bar'
+
+    assert result[1] == scopes[1]
+
+
+def test_merge_two_unrelated_on_off_scopes():
+    scopes = [_new_scope(condition='ON', test='foo'),
+              _new_scope(condition='OFF', test2='bar')]
+
+    recursive_evaluate_scope(scopes[0])
+    recursive_evaluate_scope(scopes[1])
+
+    _validate(scopes, [scopes[0]])
+
+
+def test_merge_two_unrelated_on_off_scopes():
+    scopes = [_new_scope(condition='OFF', test='foo'),
+              _new_scope(condition='ON', test2='bar')]
+
+    recursive_evaluate_scope(scopes[0])
+    recursive_evaluate_scope(scopes[1])
+
+    _validate(scopes, [scopes[1]])
+
+
+def test_merge_parent_child_scopes_with_different_conditions():
+    scope = _new_scope(condition='FOO', test1='parent')
+    scopes = [scope, _new_scope(parent_scope=scope, condition='bar', test2='child')]
+
+    recursive_evaluate_scope(scope)
+
+    _validate(scopes, scopes)
+
+
+def test_merge_parent_child_scopes_with_same_conditions():
+    scope = _new_scope(condition='FOO AND bar', test1='parent')
+    scopes = [scope, _new_scope(parent_scope=scope, condition='FOO AND bar', test2='child')]
+
+    recursive_evaluate_scope(scope)
+
+    result = merge_scopes(scopes)
+
+    assert len(result) == 1
+    r0 = result[0]
+    assert r0.parent() == None
+    assert r0.total_condition() == 'FOO AND bar'
+    assert r0.getString('test1') == 'parent'
+    assert r0.getString('test2') == 'child'
+
+
+def test_merge_parent_child_scopes_with_on_child_condition():
+    scope = _new_scope(condition='FOO AND bar', test1='parent')
+    scopes = [scope, _new_scope(parent_scope=scope, condition='ON', test2='child')]
+
+    recursive_evaluate_scope(scope)
+
+    result = merge_scopes(scopes)
+
+    assert len(result) == 1
+    r0 = result[0]
+    assert r0.parent() == None
+    assert r0.total_condition() == 'FOO AND bar'
+    assert r0.getString('test1') == 'parent'
+    assert r0.getString('test2') == 'child'
+