content_cleaner.py: add support for handling hidden files, symlinks

Add unit tests.
Flake8 and Black code formatter run.

Change-Id: I874a7410c024e7b4122087e982c1ab1861426492
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>

2 files changed, 209 insertions, 15 deletions

diff --git a/packaging-tools/content_cleaner.py b/packaging-tools/content_cleaner.py
index 8438daaf7..6922bda5b 100755
--- a/packaging-tools/content_cleaner.py
+++ b/packaging-tools/content_cleaner.py
@@ -32,11 +32,11 @@
 import os
 import sys
 import argparse
-from glob import glob
 from pathlib import Path
 from contextlib import contextmanager
 from typing import List, Generator
 from qt6_installer.logging_util import init_logger
+
 log = init_logger(__name__, debug_mode=False)
 
 
@@ -54,11 +54,23 @@ def cd(path: str) -> Generator:
         os.chdir(oldwd)
 
 
+def expand_rules(rules: List[str]) -> List[str]:
+    matches = []
+    for mask in rules:
+        # pathlib returns nothing with pattern ending "**"
+        # append "/*" to the mask for such patterns
+        mask = mask + "/*" if mask.endswith("**") else mask
+        for file in Path(".").rglob(mask):
+            matches.append(str(file))
+    return matches
+
+
 def remove_empty_directories(root_path: str):
     for root, dirs, files in os.walk(root_path, topdown=True):
         for name in dirs:
             dirPath = os.path.join(root, name)
-            if not os.listdir(dirPath):  # to check wither the dir is empty
+            if not os.listdir(dirPath):  # to check whether the dir is empty
+                log.info(f"Removing empty directory: {dirPath}")
                 os.removedirs(dirPath)
 
 
@@ -68,15 +80,13 @@ def preserve_content(input_dir: str, preserve_rules: List[str]) -> None:
         raise CleanerError(f"Not a valid input directory: {input_dir}")
     split_preserve_rules = [word for line in preserve_rules for word in line.split()]
     with cd(input_dir):
-        files_to_keep = []
-        for mask in split_preserve_rules:
-            files_to_keep.extend(glob(mask, recursive=True))
+        files_to_keep = expand_rules(split_preserve_rules)
         for p in Path(".").rglob("*"):
             if str(p) in files_to_keep:
                 continue
-            if os.path.isdir(p):
+            if not os.path.islink(p) and os.path.isdir(p):
                 continue
-            log.info(f"Removing: {p}")
+            log.info(f"Removing file: {p}")
             os.remove(p)
     remove_empty_directories(input_dir)
 
@@ -87,9 +97,7 @@ def remove_content(input_dir: str, remove_rules: List[str]) -> None:
         raise CleanerError(f"Not a valid input directory: {input_dir}")
     split_remove_rules = [word for line in remove_rules for word in line.split()]
     with cd(input_dir):
-        files_to_remove = []
-        for mask in split_remove_rules:
-            files_to_remove.extend(glob(mask, recursive=True))
+        files_to_remove = expand_rules(split_remove_rules)
         for p in Path(".").rglob("*"):
             if os.path.isdir(p):
                 continue
@@ -100,13 +108,23 @@ def remove_content(input_dir: str, remove_rules: List[str]) -> None:
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(prog="Clean all files from the --input-dir directory except those defined by --glob-rule")
+    parser = argparse.ArgumentParser(
+        prog="Clean all files from the --input-dir directory except those defined by --glob-rule"
+    )
     parser.add_argument("--input-dir", dest="input_dir", required=True, help="Directory to scan")
     group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("--preserve", dest="preserve_rules", action='append',
-                        help="One or multiple glob based rules which files to keep")
-    group.add_argument("--remove", dest="remove_rules", action='append',
-                        help="One or multiple glob based rules which files to remove")
+    group.add_argument(
+        "--preserve",
+        dest="preserve_rules",
+        action="append",
+        help="One or multiple glob based rules which files to keep",
+    )
+    group.add_argument(
+        "--remove",
+        dest="remove_rules",
+        action="append",
+        help="One or multiple glob based rules which files to remove",
+    )
     args = parser.parse_args(sys.argv[1:])
     if args.preserve_rules:
         preserve_content(args.input_dir, args.preserve_rules)
diff --git a/packaging-tools/test_content_cleaner.py b/packaging-tools/test_content_cleaner.py
new file mode 100644
index 000000000..add54789c
--- /dev/null
+++ b/packaging-tools/test_content_cleaner.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#############################################################################
+##
+## Copyright (C) 2021 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the release tools of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+
+import os
+import unittest
+import tempfile
+from typing import List
+from ddt import ddt, data, unpack
+from content_cleaner import preserve_content, remove_content, remove_empty_directories
+
+
+@ddt
+class TestContentCleaner(unittest.TestCase):
+    def generate_test_content(self, test_base_dir: str, test_content_paths: List[str]):
+        for test_content_path in test_content_paths:
+            test_path = os.path.join(test_base_dir, test_content_path)
+            os.makedirs(os.path.dirname(test_path), exist_ok=True)
+            if not test_path.endswith("/"):
+                with open(test_path, "w+") as f:
+                    f.write("")
+
+    @data(
+        (
+            [
+                "libexec/codegen/preserve-file.ext",
+                "libexec/codegen/test-file1",
+                "libexec/codegen/test-file2",
+            ],  # test content
+            ["libexec/codegen/preserve-file.ext"],  # expected result
+            ["preserve-file.ext"],  # preserve rule
+        ),
+        (
+            [
+                "libexec/codegen/test-file1",
+                "libexec/codegen/test-file2",
+                "libexec/codegen/test-file3",
+                "remove/this/path/",
+                "remove/this/path/test-file1",
+            ],
+            [
+                "libexec/codegen/test-file1",
+                "libexec/codegen/test-file1",
+                "libexec/codegen/test-file2",
+                "libexec/codegen/test-file3",
+            ],
+            ["libexec/codegen/**"],
+        ),
+        (
+            ["bin/test-file1", "bin/test-file2", "bin/test-file3", "remove/this/path/.test-file1"],
+            ["bin/test-file1", "bin/test-file2", "bin/test-file3"],
+            ["bin/*"],
+        ),
+        (
+            [
+                "libexec/codegen/preserve-file.ext",
+                "libexec/codegen/.preserve-file",
+                "libexec/codegen/remove-file",
+                "remove/this/path/",
+                "bin/codegen/.preserve-file",
+            ],
+            [
+                "libexec/codegen/preserve-file.ext",
+                "libexec/codegen/.preserve-file",
+                "bin/codegen/.preserve-file",
+            ],
+            ["**/*preserve-file*"],
+        ),
+    )
+    @unpack
+    def test_preserve_content(
+        self,
+        test_content: List[str],
+        expected_result: List[str],
+        preserve_rules: List[str],
+    ) -> None:
+        with tempfile.TemporaryDirectory(dir=os.getcwd()) as tmp_base_dir:
+            test_base_dir = os.path.join(tmp_base_dir, "test-base-dir")
+            self.generate_test_content(test_base_dir, test_content)
+            preserve_content(test_base_dir, preserve_rules)
+            for item in expected_result:
+                self.assertTrue(os.path.isfile(os.path.join(test_base_dir, item)))
+
+    @data(
+        (
+            [
+                "testA/path1/path2/path3/testfile1.ext",
+                "testA/path1/path2/path3/.testfile2",
+                "testA/path1/path2/path3/testfile3",
+                "testB/path3/path4/testfile1.ext",
+                "testB/path3/path4/.testfile2",
+                "testB/path5/path5/testfile3",
+            ],
+            [
+                "**/path1/path2/**/testfile1.ext",
+                "**/path1/path2/path3/.testfile2",
+                "**/path3/path4/testfile1.ext",
+                "**/path3/path4/.testfile2",
+            ],
+            ["testfile1.ext", ".testfile2"],
+        ),
+        (
+            [
+                "testA/path1/path2/path3/testfile1.ext",
+                "testA/path1/path2/path3/.testfile2",
+                "testA/path1/path2/path3/testfile3",
+            ],
+            ["**/*testfile*"],
+            ["testfile1.ext", ".testfile2", "testfile3"],
+        ),
+    )
+    @unpack
+    def test_remove_content(
+        self,
+        test_content: List[str],
+        remove_rules: List[str],
+        verify_removed_files: List[str],
+    ) -> None:
+
+        with tempfile.TemporaryDirectory(dir=os.getcwd()) as tmp_base_dir:
+            test_base_dir = os.path.join(tmp_base_dir, "test-base-dir")
+            self.generate_test_content(test_base_dir, test_content)
+            remove_content(test_base_dir, remove_rules)
+            for file_path in verify_removed_files:
+                for path in test_content:
+                    if file_path in os.path.join(test_base_dir, path):
+                        self.assertFalse(os.path.isfile(os.path.join(test_base_dir, path)))
+
+    @data(
+        (["test/path/test-file", "test/path/.test-file"], False),
+        (["test/path/to/remove/", "test/.path/to/remove/"], True),
+    )
+    @unpack
+    def test_remove_empty_directories(self, test_content: str, remove_dir: bool) -> None:
+        with tempfile.TemporaryDirectory(dir=os.getcwd()) as tmp_base_dir:
+            test_base_dir = os.path.join(tmp_base_dir, "test-base-dir")
+            self.generate_test_content(test_base_dir, test_content)
+            remove_empty_directories(test_base_dir)
+            for path in test_content:
+                verify_path = os.path.join(test_base_dir, path)
+                if remove_dir:
+                    self.assertFalse(os.path.exists(verify_path))
+                else:
+                    self.assertTrue(os.path.exists(verify_path))
+
+
+if __name__ == "__main__":
+    unittest.main()