Speedup run_pro2cmake

We can use all cores. Sadly it doesn't balance cores well as
corelib.pro takes most of the time anyway, but the speedup is
visible from ~15 to 5 min.

Change-Id: Id8209c58491b38d19c6e9f1163d366c3e33a182c
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
Reviewed-by: Tobias Hunger <tobias.hunger@qt.io>

1 files changed, 23 insertions, 11 deletions

diff --git a/util/cmake/run_pro2cmake.py b/util/cmake/run_pro2cmake.py
index 1a3c1581e3..b3a07c7522 100755
--- a/util/cmake/run_pro2cmake.py
+++ b/util/cmake/run_pro2cmake.py
@@ -30,7 +30,9 @@
 import glob
 import os
 import subprocess
+import concurrent.futures
 import sys
+import typing
 
 script_path = os.path.dirname(os.path.abspath(__file__))
 base_path = os.path.dirname(script_path)
@@ -41,20 +43,30 @@ if len(sys.argv) > 1:
 
 failed_files = []
 
-pro_file_count = 0
-for filename in glob.iglob(os.path.join(base_path, '**/*.pro'),
-                           recursive=True):
-    pro_file_count += 1
-    print('{} ({}): Converting: {} ...'
-          .format(pro_file_count, len(failed_files), filename))
-    result = subprocess.run([pro2cmake, os.path.basename(filename)],
-                            cwd=os.path.dirname(filename))
-    if result.returncode != 0:
-        failed_files.append(filename)
+all_files = glob.glob(os.path.join(base_path, '**/*.pro'), recursive=True)
+files_count = len(all_files)
+
+with concurrent.futures.ThreadPoolExecutor(initializer=os.nice, initargs=(10,)) as pool:
+
+    def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]:
+        filename, index, total = data
+        result = subprocess.run((pro2cmake, os.path.basename(filename)),
+                                cwd=os.path.dirname(filename),
+                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename)
+        return result.returncode, filename, stdout + result.stdout.decode()
+
+    for return_code, filename, stdout in pool.map(_process_a_file,
+                                                  zip(all_files,
+                                                      range(1, files_count + 1),
+                                                      (files_count for _ in all_files))):
+        if return_code:
+            failed_files.append(filename)
+        print(stdout)
 
 if failed_files:
     print('The following files were not successfully '
-          'converted ({} of {}):'.format(len(failed_files), pro_file_count))
+          'converted ({} of {}):'.format(len(failed_files), files_count))
     for f in failed_files:
         print('    "{}"'.format(f))