diff options
author | Alexandru Croitor <alexandru.croitor@qt.io> | 2019-10-09 17:27:17 +0200 |
---|---|---|
committer | Alexandru Croitor <alexandru.croitor@qt.io> | 2019-10-10 10:04:50 +0000 |
commit | fbace1f4e059dee498332146da0d90c627886952 (patch) | |
tree | 2cf696c75ef2d377d285159c1fa9016a80efffcf | |
parent | f863bf7d6452740e131e3e846922d57cab2173f6 (diff) |
pro2cmake: Make condition cache work well with run_pro2cmake
When using run_pro2cmake, multiple pro2cmake processes try to access
and override the condition cache.
Make sure that reads / writes of the cache file are protected by
a file lock, and the content is merged rather than overridden.
This requires use of a new pip package, portalocker.
The script will tell the user to install it if it's missing.
Change-Id: I44798c46ff0912981b186bec40e3e918f249fb4d
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>
-rw-r--r-- | util/cmake/condition_simplifier_cache.py | 75 | ||||
-rw-r--r-- | util/cmake/requirements.txt | 1 |
2 files changed, 70 insertions, 6 deletions
diff --git a/util/cmake/condition_simplifier_cache.py b/util/cmake/condition_simplifier_cache.py index 947b18af81..6303ff828e 100644 --- a/util/cmake/condition_simplifier_cache.py +++ b/util/cmake/condition_simplifier_cache.py @@ -35,7 +35,7 @@ import os import sys import time -from typing import Any, Callable, Dict, Union +from typing import Any, Callable, Dict condition_simplifier_cache_enabled = True @@ -86,16 +86,49 @@ def init_cache_dict(): } +def merge_dicts_recursive(a: Dict[str, Any], other: Dict[str, Any]) -> Dict[str, Any]: + """Merges values of "other" into "a", mutates a.""" + for key in other: + if key in a: + if isinstance(a[key], dict) and isinstance(other[key], dict): + merge_dicts_recursive(a[key], other[key]) + elif a[key] == other[key]: + pass + else: + a[key] = other[key] + return a + + +def open_file_safe(file_path: str, mode: str = "r+"): + # Use portalocker package for file locking if available, + # otherwise print a message to install the package. + try: + import portalocker # type: ignore + + file_open_func = portalocker.Lock + file_open_args = [file_path] + file_open_kwargs = {"mode": mode, "flags": portalocker.LOCK_EX} + file_handle = file_open_func(*file_open_args, **file_open_kwargs) + return file_handle + except ImportError: + print( + "The conversion script is missing a required package: portalocker. Please run " + "python -m pip install requirements.txt to install the missing dependency." + ) + exit(1) + + def simplify_condition_memoize(f: Callable[[str], str]): cache_path = get_cache_location() cache_file_content: Dict[str, Any] = {} if os.path.exists(cache_path): try: - with open(cache_path, "r") as cache_file: + with open_file_safe(cache_path, mode="r") as cache_file: cache_file_content = json.load(cache_file) except (IOError, ValueError): - pass + print(f"Invalid pro2cmake cache file found at: {cache_path}. Removing it.") + os.remove(cache_path) if not cache_file_content: cache_file_content = init_cache_dict() @@ -107,13 +140,43 @@ def simplify_condition_memoize(f: Callable[[str], str]): def update_cache_file(): if not os.path.exists(cache_path): os.makedirs(os.path.dirname(cache_path), exist_ok=True) - with open(cache_path, "w") as cache_file_write_handle: - json.dump(cache_file_content, cache_file_write_handle, indent=4) + # Create the file if it doesn't exist, but don't override + # it. + with open(cache_path, "a") as temp_file_handle: + pass + + updated_cache = cache_file_content + + with open_file_safe(cache_path, "r+") as cache_file_write_handle: + # Read any existing cache content, and truncate the file. + cache_file_existing_content = cache_file_write_handle.read() + cache_file_write_handle.seek(0) + cache_file_write_handle.truncate() + + # Merge the new cache into the old cache if it exists. + if cache_file_existing_content: + possible_cache = json.loads(cache_file_existing_content) + if ( + "checksum" in possible_cache + and "schema_version" in possible_cache + and possible_cache["checksum"] == cache_file_content["checksum"] + and possible_cache["schema_version"] == cache_file_content["schema_version"] + ): + updated_cache = merge_dicts_recursive(dict(possible_cache), updated_cache) + + json.dump(updated_cache, cache_file_write_handle, indent=4) + + # Flush any buffered writes. + cache_file_write_handle.flush() + os.fsync(cache_file_write_handle.fileno()) atexit.register(update_cache_file) def helper(condition: str) -> str: - if condition not in cache_file_content["cache"]["conditions"] or not condition_simplifier_cache_enabled: + if ( + condition not in cache_file_content["cache"]["conditions"] + or not condition_simplifier_cache_enabled + ): cache_file_content["cache"]["conditions"][condition] = f(condition) return cache_file_content["cache"]["conditions"][condition] diff --git a/util/cmake/requirements.txt b/util/cmake/requirements.txt index 3a5d19a044..4d42912956 100644 --- a/util/cmake/requirements.txt +++ b/util/cmake/requirements.txt @@ -2,3 +2,4 @@ pytest; python_version >= '3.7' mypy; python_version >= '3.7' pyparsing; python_version >= '3.7' sympy; python_version >= '3.7' +portalocker; python_version >= '3.7' |