From eb3d73ffb778e191aa5705fd1867c62809eedf9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Nowacki?= Date: Mon, 25 Mar 2019 15:16:32 +0100 Subject: Speedup run_pro2cmake We can use all cores. Sadly it doesn't balance cores well as corelib.pro takes most of the time anyway, but the speedup is visible from ~15 to 5 min. Change-Id: Id8209c58491b38d19c6e9f1163d366c3e33a182c Reviewed-by: Simon Hausmann Reviewed-by: Tobias Hunger --- util/cmake/run_pro2cmake.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/util/cmake/run_pro2cmake.py b/util/cmake/run_pro2cmake.py index 1a3c1581e3..b3a07c7522 100755 --- a/util/cmake/run_pro2cmake.py +++ b/util/cmake/run_pro2cmake.py @@ -30,7 +30,9 @@ import glob import os import subprocess +import concurrent.futures import sys +import typing script_path = os.path.dirname(os.path.abspath(__file__)) base_path = os.path.dirname(script_path) @@ -41,20 +43,30 @@ if len(sys.argv) > 1: failed_files = [] -pro_file_count = 0 -for filename in glob.iglob(os.path.join(base_path, '**/*.pro'), - recursive=True): - pro_file_count += 1 - print('{} ({}): Converting: {} ...' - .format(pro_file_count, len(failed_files), filename)) - result = subprocess.run([pro2cmake, os.path.basename(filename)], - cwd=os.path.dirname(filename)) - if result.returncode != 0: - failed_files.append(filename) +all_files = glob.glob(os.path.join(base_path, '**/*.pro'), recursive=True) +files_count = len(all_files) + +with concurrent.futures.ThreadPoolExecutor(initializer=os.nice, initargs=(10,)) as pool: + + def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]: + filename, index, total = data + result = subprocess.run((pro2cmake, os.path.basename(filename)), + cwd=os.path.dirname(filename), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename) + return result.returncode, filename, stdout + result.stdout.decode() + + for return_code, filename, stdout in pool.map(_process_a_file, + zip(all_files, + range(1, files_count + 1), + (files_count for _ in all_files))): + if return_code: + failed_files.append(filename) + print(stdout) if failed_files: print('The following files were not successfully ' - 'converted ({} of {}):'.format(len(failed_files), pro_file_count)) + 'converted ({} of {}):'.format(len(failed_files), files_count)) for f in failed_files: print(' "{}"'.format(f)) -- cgit v1.2.3