#!/usr/bin/env python3 ############################################################################# ## ## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the plugins of the Qt Toolkit. ## ## $QT_BEGIN_LICENSE:GPL-EXCEPT$ ## Commercial License Usage ## Licensees holding valid commercial Qt licenses may use this file in ## accordance with the commercial license agreement provided with the ## Software or, alternatively, in accordance with the terms contained in ## a written agreement between you and The Qt Company. For licensing terms ## and conditions see https://www.qt.io/terms-conditions. For further ## information use the contact form at https://www.qt.io/contact-us. ## ## GNU General Public License Usage ## Alternatively, this file may be used under the terms of the GNU ## General Public License version 3 as published by the Free Software ## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT ## included in the packaging of this file. Please review the following ## information to ensure the GNU General Public License requirements will ## be met: https://www.gnu.org/licenses/gpl-3.0.html. ## ## $QT_END_LICENSE$ ## ############################################################################# import collections import os import re from itertools import chain from typing import Tuple import pyparsing as pp # type: ignore from helper import _set_up_py_parsing_nicer_debug_output _set_up_py_parsing_nicer_debug_output(pp) def fixup_linecontinuation(contents: str) -> str: # Remove all line continuations, aka a backslash followed by # a newline character with an arbitrary amount of whitespace # between the backslash and the newline. # This greatly simplifies the qmake parsing grammar. contents = re.sub(r"([^\t ])\\[ \t]*\n", "\\1 ", contents) contents = re.sub(r"\\[ \t]*\n", "", contents) return contents def fixup_comments(contents: str) -> str: # Get rid of completely commented out lines. # So any line which starts with a '#' char and ends with a new line # will be replaced by a single new line. # # This is needed because qmake syntax is weird. In a multi line # assignment (separated by backslashes and newlines aka # # \\\n ), if any of the lines are completely commented out, in # principle the assignment should fail. # # It should fail because you would have a new line separating # the previous value from the next value, and the next value would # not be interpreted as a value, but as a new token / operation. # qmake is lenient though, and accepts that, so we need to take # care of it as well, as if the commented line didn't exist in the # first place. contents = re.sub(r"\n#[^\n]*?\n", "\n", contents, re.DOTALL) return contents def flatten_list(l): """ Flattens an irregular nested list into a simple list.""" for el in l: if isinstance(el, collections.abc.Iterable) and not isinstance(el, (str, bytes)): yield from flatten_list(el) else: yield el def handle_function_value(group: pp.ParseResults): function_name = group[0] function_args = group[1] if function_name == "qtLibraryTarget": if len(function_args) > 1: raise RuntimeError( "Don't know what to with more than one function argument " "for $$qtLibraryTarget()." ) return str(function_args[0]) if function_name == "quote": # Do nothing, just return a string result return str(group) if function_name == "files": return str(function_args[0]) if function_name == "basename": if len(function_args) != 1: print(f"XXXX basename with more than one argument") if function_args[0] == "_PRO_FILE_PWD_": return os.path.basename(os.getcwd()) print(f"XXXX basename with value other than _PRO_FILE_PWD_") return os.path.basename(str(function_args[0])) if isinstance(function_args, pp.ParseResults): function_args = list(flatten_list(function_args.asList())) # For other functions, return the whole expression as a string. return f"$${function_name}({' '.join(function_args)})" class QmakeParser: def __init__(self, *, debug: bool = False) -> None: self.debug = debug self._Grammar = self._generate_grammar() def _generate_grammar(self): # Define grammar: pp.ParserElement.setDefaultWhitespaceChars(" \t") def add_element(name: str, value: pp.ParserElement): nonlocal self if self.debug: value.setName(name) value.setDebug() return value EOL = add_element("EOL", pp.Suppress(pp.LineEnd())) Else = add_element("Else", pp.Keyword("else")) Identifier = add_element( "Identifier", pp.Word(f"{pp.alphas}_", bodyChars=pp.alphanums + "_-./") ) BracedValue = add_element( "BracedValue", pp.nestedExpr( ignoreExpr=pp.quotedString | pp.QuotedString( quoteChar="$(", endQuoteChar=")", escQuote="\\", unquoteResults=False ) ).setParseAction(lambda s, l, t: ["(", *t[0], ")"]), ) Substitution = add_element( "Substitution", pp.Combine( pp.Literal("$") + ( ( (pp.Literal("$") + Identifier + pp.Optional(pp.nestedExpr())) | (pp.Literal("(") + Identifier + pp.Literal(")")) | (pp.Literal("{") + Identifier + pp.Literal("}")) | ( pp.Literal("$") + pp.Literal("{") + Identifier + pp.Optional(pp.nestedExpr()) + pp.Literal("}") ) | (pp.Literal("$") + pp.Literal("[") + Identifier + pp.Literal("]")) ) ) ), ) LiteralValuePart = add_element( "LiteralValuePart", pp.Word(pp.printables, excludeChars="$#{}()") ) SubstitutionValue = add_element( "SubstitutionValue", pp.Combine(pp.OneOrMore(Substitution | LiteralValuePart | pp.Literal("$"))), ) FunctionValue = add_element( "FunctionValue", pp.Group( pp.Suppress(pp.Literal("$") + pp.Literal("$")) + Identifier + pp.nestedExpr() # .setParseAction(lambda s, l, t: ['(', *t[0], ')']) ).setParseAction(lambda s, l, t: handle_function_value(*t)), ) Value = add_element( "Value", pp.NotAny(Else | pp.Literal("}") | EOL) + ( pp.QuotedString(quoteChar='"', escChar="\\") | FunctionValue | SubstitutionValue | BracedValue ), ) Values = add_element("Values", pp.ZeroOrMore(Value)("value")) Op = add_element( "OP", pp.Literal("=") | pp.Literal("-=") | pp.Literal("+=") | pp.Literal("*=") | pp.Literal("~="), ) Key = add_element("Key", Identifier) Operation = add_element( "Operation", Key("key") + pp.locatedExpr(Op)("operation") + Values("value") ) CallArgs = add_element("CallArgs", pp.nestedExpr()) def parse_call_args(results): out = "" for item in chain(*results): if isinstance(item, str): out += item else: out += "(" + parse_call_args(item) + ")" return out CallArgs.setParseAction(parse_call_args) Load = add_element("Load", pp.Keyword("load") + CallArgs("loaded")) Include = add_element( "Include", pp.Keyword("include") + pp.locatedExpr(CallArgs)("included") ) Option = add_element("Option", pp.Keyword("option") + CallArgs("option")) RequiresCondition = add_element("RequiresCondition", pp.originalTextFor(pp.nestedExpr())) def parse_requires_condition(s, l, t): # The following expression unwraps the condition via the additional info # set by originalTextFor. condition_without_parentheses = s[t._original_start + 1 : t._original_end - 1] # And this replaces the colons with '&&' similar how it's done for 'Condition'. condition_without_parentheses = ( condition_without_parentheses.strip().replace(":", " && ").strip(" && ") ) return condition_without_parentheses RequiresCondition.setParseAction(parse_requires_condition) Requires = add_element( "Requires", pp.Keyword("requires") + RequiresCondition("project_required_condition") ) # ignore the whole thing... DefineTestDefinition = add_element( "DefineTestDefinition", pp.Suppress( pp.Keyword("defineTest") + CallArgs + pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd()) ), ) # ignore the whole thing... ForLoop = add_element( "ForLoop", pp.Suppress( pp.Keyword("for") + CallArgs + pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd()) ), ) # ignore the whole thing... ForLoopSingleLine = add_element( "ForLoopSingleLine", pp.Suppress(pp.Keyword("for") + CallArgs + pp.Literal(":") + pp.SkipTo(EOL)), ) # ignore the whole thing... FunctionCall = add_element("FunctionCall", pp.Suppress(Identifier + pp.nestedExpr())) Scope = add_element("Scope", pp.Forward()) Statement = add_element( "Statement", pp.Group( Load | Include | Option | Requires | ForLoop | ForLoopSingleLine | DefineTestDefinition | FunctionCall | Operation ), ) StatementLine = add_element("StatementLine", Statement + (EOL | pp.FollowedBy("}"))) StatementGroup = add_element( "StatementGroup", pp.ZeroOrMore(StatementLine | Scope | pp.Suppress(EOL)) ) Block = add_element( "Block", pp.Suppress("{") + pp.Optional(EOL) + StatementGroup + pp.Optional(EOL) + pp.Suppress("}") + pp.Optional(EOL), ) ConditionEnd = add_element( "ConditionEnd", pp.FollowedBy( (pp.Optional(pp.White()) + (pp.Literal(":") | pp.Literal("{") | pp.Literal("|"))) ), ) ConditionPart1 = add_element( "ConditionPart1", (pp.Optional("!") + Identifier + pp.Optional(BracedValue)) ) ConditionPart2 = add_element("ConditionPart2", pp.CharsNotIn("#{}|:=\\\n")) ConditionPart = add_element( "ConditionPart", (ConditionPart1 ^ ConditionPart2) + ConditionEnd ) ConditionOp = add_element("ConditionOp", pp.Literal("|") ^ pp.Literal(":")) ConditionWhiteSpace = add_element( "ConditionWhiteSpace", pp.Suppress(pp.Optional(pp.White(" "))) ) ConditionRepeated = add_element( "ConditionRepeated", pp.ZeroOrMore(ConditionOp + ConditionWhiteSpace + ConditionPart) ) Condition = add_element("Condition", pp.Combine(ConditionPart + ConditionRepeated)) Condition.setParseAction(lambda x: " ".join(x).strip().replace(":", " && ").strip(" && ")) # Weird thing like write_file(a)|error() where error() is the alternative condition # which happens to be a function call. In this case there is no scope, but our code expects # a scope with a list of statements, so create a fake empty statement. ConditionEndingInFunctionCall = add_element( "ConditionEndingInFunctionCall", pp.Suppress(ConditionOp) + FunctionCall + pp.Empty().setParseAction(lambda x: [[]]).setResultsName("statements"), ) SingleLineScope = add_element( "SingleLineScope", pp.Suppress(pp.Literal(":")) + pp.Group(Block | (Statement + EOL))("statements"), ) MultiLineScope = add_element("MultiLineScope", Block("statements")) SingleLineElse = add_element( "SingleLineElse", pp.Suppress(pp.Literal(":")) + (Scope | Block | (Statement + pp.Optional(EOL))), ) MultiLineElse = add_element("MultiLineElse", Block) ElseBranch = add_element("ElseBranch", pp.Suppress(Else) + (SingleLineElse | MultiLineElse)) # Scope is already add_element'ed in the forward declaration above. Scope <<= pp.Group( Condition("condition") + (SingleLineScope | MultiLineScope | ConditionEndingInFunctionCall) + pp.Optional(ElseBranch)("else_statements") ) Grammar = StatementGroup("statements") Grammar.ignore(pp.pythonStyleComment()) return Grammar def parseFile(self, file: str) -> Tuple[pp.ParseResults, str]: print(f'Parsing "{file}"...') try: with open(file, "r") as file_fd: contents = file_fd.read() # old_contents = contents contents = fixup_comments(contents) contents = fixup_linecontinuation(contents) result = self._Grammar.parseString(contents, parseAll=True) except pp.ParseException as pe: print(pe.line) print(f"{' ' * (pe.col-1)}^") print(pe) raise pe return result, contents def parseProFile(file: str, *, debug=False) -> Tuple[pp.ParseResults, str]: parser = QmakeParser(debug=debug) return parser.parseFile(file)