diff options
Diffstat (limited to 'sources/pyside-tools/qtpy2cpp_lib/tokenizer.py')
-rw-r--r-- | sources/pyside-tools/qtpy2cpp_lib/tokenizer.py | 55 |
1 files changed, 55 insertions, 0 deletions
diff --git a/sources/pyside-tools/qtpy2cpp_lib/tokenizer.py b/sources/pyside-tools/qtpy2cpp_lib/tokenizer.py new file mode 100644 index 000000000..d5e26c2a8 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/tokenizer.py @@ -0,0 +1,55 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""Tool to dump Python Tokens""" + + +import sys +import tokenize + + +def format_token(t): + r = repr(t) + if r.startswith('TokenInfo('): + r = r[10:] + pos = r.find("), line='") + if pos < 0: + pos = r.find('), line="') + if pos > 0: + r = r[:pos + 1] + return r + + +def first_non_space(s): + for i, c in enumerate(s): + if c != ' ': + return i + return 0 + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Specify file Name") + sys.exit(1) + filename = sys.argv[1] + indent_level = 0 + indent = '' + last_line_number = -1 + with tokenize.open(filename) as f: + generator = tokenize.generate_tokens(f.readline) + for t in generator: + line_number = t.start[0] + if line_number != last_line_number: + code_line = t.line.rstrip() + non_space = first_non_space(code_line) + print('{:04d} {}{}'.format(line_number, '_' * non_space, + code_line[non_space:])) + last_line_number = line_number + if t.type == tokenize.INDENT: + indent_level = indent_level + 1 + indent = ' ' * indent_level + elif t.type == tokenize.DEDENT: + indent_level = indent_level - 1 + indent = ' ' * indent_level + else: + print(' ', indent, format_token(t)) |