Improve configurejson2cmake.py to handle non-compliant qmake JSON

Some configure.json files contain new lines inside quoted strings,
which is not conformant with the JSON spec.

Add a new json_parser python module which uses pyparsing to preprocess
the json files to remove the new lines inside the quoted strings, and
then hands over the preprocessed content to the regular json module.

Change-Id: I5f8938492068dda5640465cc78f5a7b6be0e709a
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
Reviewed-by: Qt CMake Build Bot

4 files changed, 131 insertions, 32 deletions

diff --git a/util/cmake/configurejson2cmake.py b/util/cmake/configurejson2cmake.py
index d1646ed082..09f76a272d 100755
--- a/util/cmake/configurejson2cmake.py
+++ b/util/cmake/configurejson2cmake.py
@@ -27,7 +27,7 @@
 ##
 #############################################################################
 
-import json
+import json_parser
 import os.path
 import re
 import sys
@@ -154,8 +154,8 @@ def readJsonFromDir(dir):
     print('Reading {}...'.format(path))
     assert os.path.exists(path)
 
-    with open(path, 'r') as fh:
-        return json.load(fh)
+    parser = json_parser.QMakeSpecificJSONParser()
+    return parser.parse(path)
 
 
 def processFiles(ctx, data):
diff --git a/util/cmake/helper.py b/util/cmake/helper.py
index 682e2ec15f..b7d91921fa 100644
--- a/util/cmake/helper.py
+++ b/util/cmake/helper.py
@@ -428,3 +428,29 @@ def generate_find_package_info(lib: LibraryMapping,
                                                  ind=one_ind)
 
     return result
+
+
+def _set_up_py_parsing_nicer_debug_output(pp):
+    indent = -1
+
+    def increase_indent(fn):
+        def wrapper_function(*args):
+            nonlocal indent
+            indent += 1
+            print("> " * indent, end="")
+            return fn(*args)
+
+        return wrapper_function
+
+    def decrease_indent(fn):
+        def wrapper_function(*args):
+            nonlocal indent
+            print("> " * indent, end="")
+            indent -= 1
+            return fn(*args)
+
+        return wrapper_function
+
+    pp._defaultStartDebugAction = increase_indent(pp._defaultStartDebugAction)
+    pp._defaultSuccessDebugAction = decrease_indent(pp._defaultSuccessDebugAction)
+    pp._defaultExceptionDebugAction = decrease_indent(pp._defaultExceptionDebugAction)
diff --git a/util/cmake/json_parser.py b/util/cmake/json_parser.py
new file mode 100644
index 0000000000..6ead008f08
--- /dev/null
+++ b/util/cmake/json_parser.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+#############################################################################
+##
+## Copyright (C) 2019 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the plugins of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+import pyparsing as pp
+import json
+import re
+from helper import _set_up_py_parsing_nicer_debug_output
+_set_up_py_parsing_nicer_debug_output(pp)
+
+
+class QMakeSpecificJSONParser:
+    def __init__(self, *, debug: bool = False) -> None:
+        self.debug = debug
+        self.grammar = self.create_py_parsing_grammar()
+
+    def create_py_parsing_grammar(self):
+        # Keep around all whitespace.
+        pp.ParserElement.setDefaultWhitespaceChars('')
+
+        def add_element(name: str, value: pp.ParserElement):
+            nonlocal self
+            if self.debug:
+                value.setName(name)
+                value.setDebug()
+            return value
+
+        # Our grammar is pretty simple. We want to remove all newlines
+        # inside quoted strings, to make the quoted strings JSON
+        # compliant. So our grammar should skip to the first quote while
+        # keeping everything before it as-is, process the quoted string
+        # skip to the next quote, and repeat that until the end of the
+        # file.
+
+        EOF = add_element('EOF', pp.StringEnd())
+        SkipToQuote = add_element('SkipToQuote', pp.SkipTo('"'))
+        SkipToEOF = add_element('SkipToEOF', pp.SkipTo(EOF))
+
+        def remove_newlines_and_whitespace_in_quoted_string(tokens):
+            first_string = tokens[0]
+            replaced_string = re.sub(r'\n[ ]*', ' ', first_string)
+            return replaced_string
+
+        QuotedString = add_element('QuotedString', pp.QuotedString(quoteChar='"',
+                                                                   multiline=True,
+                                                                   unquoteResults=False))
+        QuotedString.setParseAction(remove_newlines_and_whitespace_in_quoted_string)
+
+        QuotedTerm = add_element('QuotedTerm', pp.Optional(SkipToQuote) + QuotedString)
+        Grammar = add_element('Grammar', pp.OneOrMore(QuotedTerm) + SkipToEOF)
+
+        return Grammar
+
+    def parse_file_using_py_parsing(self, file: str):
+        print('Pre processing "{}" using py parsing to remove incorrect newlines.'.format(file))
+        try:
+            with open(file, 'r') as file_fd:
+                contents = file_fd.read()
+
+            parser_result = self.grammar.parseString(contents, parseAll=True)
+            token_list = parser_result.asList()
+            joined_string = ''.join(token_list)
+
+            return joined_string
+        except pp.ParseException as pe:
+            print(pe.line)
+            print(' '*(pe.col-1) + '^')
+            print(pe)
+            raise pe
+
+    def parse(self, file: str):
+        pre_processed_string = self.parse_file_using_py_parsing(file)
+        print('Parsing "{}" using json.loads().'.format(file))
+        json_parsed = json.loads(pre_processed_string)
+        return json_parsed
diff --git a/util/cmake/pro2cmake.py b/util/cmake/pro2cmake.py
index 5a2db68e67..7c2a625489 100755
--- a/util/cmake/pro2cmake.py
+++ b/util/cmake/pro2cmake.py
@@ -41,6 +41,8 @@ import typing
 
 from sympy.logic import (simplify_logic, And, Or, Not,)
 import pyparsing as pp
+from helper import _set_up_py_parsing_nicer_debug_output
+_set_up_py_parsing_nicer_debug_output(pp)
 
 from helper import map_qt_library, map_3rd_party_library, is_known_3rd_party_library, \
     featureName, map_platform, find_library_info_for_target, generate_find_package_info, \
@@ -687,32 +689,6 @@ class QmakeParser:
         self.debug = debug
         self._Grammar = self._generate_grammar()
 
-    @staticmethod
-    def set_up_py_parsing_nicer_debug_output():
-        indent = -1
-
-        def increase_indent(fn):
-            def wrapper_function(*args):
-                nonlocal indent
-                indent += 1
-                print("> " * indent, end="")
-                return fn(*args)
-
-            return wrapper_function
-
-        def decrease_indent(fn):
-            def wrapper_function(*args):
-                nonlocal indent
-                print("> " * indent, end="")
-                indent -= 1
-                return fn(*args)
-
-            return wrapper_function
-
-        pp._defaultStartDebugAction = increase_indent(pp._defaultStartDebugAction)
-        pp._defaultSuccessDebugAction = decrease_indent(pp._defaultSuccessDebugAction)
-        pp._defaultExceptionDebugAction = decrease_indent(pp._defaultExceptionDebugAction)
-
     def _generate_grammar(self):
         # Define grammar:
         pp.ParserElement.setDefaultWhitespaceChars(' \t')
@@ -900,9 +876,6 @@ class QmakeParser:
         return result
 
 
-QmakeParser.set_up_py_parsing_nicer_debug_output()
-
-
 def parseProFile(file: str, *, debug=False):
     parser = QmakeParser(debug=debug)
     return parser.parseFile(file)