snippets_translate: Add a way of using snippets from Python

Split a helper off the snippet extraction function taking the comment
pattern and let it return a dict by id to make it possible to replace
snippets by id.

Prototypically use it for the  modelview tutorial.

Fixes: PYSIDE-1984
Task-number: PYSIDE-1952
Pick-to: 6.3 6.2
Change-Id: I05dbc3e36825761fe2968d6507880cd6f588682d
Reviewed-by: Christian Tismer <tismer@stackless.com>

3 files changed, 140 insertions, 29 deletions

diff --git a/tools/snippets_translate/main.py b/tools/snippets_translate/main.py
index a0743e013..3f8152a74 100644
--- a/tools/snippets_translate/main.py
+++ b/tools/snippets_translate/main.py
@@ -5,14 +5,28 @@ import logging
 import os
 import re
 import sys
-from argparse import ArgumentParser, Namespace
+from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
 from enum import Enum
 from pathlib import Path
 from textwrap import dedent
-from typing import List
+from typing import Dict, List
 
+from override import python_example_snippet_mapping
 from converter import snippet_translate
 
+HELP = """Converts Qt C++ code snippets to Python snippets.
+
+Ways to override Snippets:
+
+1) Complete snippets from local files:
+   To replace snippet "[1]" of "foo/bar.cpp", create a file
+   "sources/pyside6/doc/snippets/foo/bar_1.cpp.py" .
+2) Snippets extracted from Python examples:
+   To use snippets from Python examples, add markers ("#! [id]") to it
+   and an entry to _PYTHON_EXAMPLE_SNIPPET_MAPPING.
+"""
+
+
 # Logger configuration
 try:
     from rich.logging import RichHandler
@@ -37,11 +51,16 @@ log = logging.getLogger("snippets_translate")
 # Filter and paths configuration
 SKIP_END = (".pro", ".pri", ".cmake", ".qdoc", ".yaml", ".frag", ".qsb", ".vert", "CMakeLists.txt")
 SKIP_BEGIN = ("changes-", ".")
-SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]")
+CPP_SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]")
+PYTHON_SNIPPET_PATTERN = re.compile(r"#! ?\[([^]]+)\]")
+
+ROOT_PATH = Path(__file__).parents[2]
+SOURCE_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "snippets"
+
 
-SOURCE_PATH = Path(__file__).parents[2] / "sources" / "pyside6" / "doc" / "snippets"
 OVERRIDDEN_SNIPPET = "# OVERRIDDEN_SNIPPET"
 
+
 class FileStatus(Enum):
     Exists = 0
     New = 1
@@ -52,7 +71,9 @@ def get_parser() -> ArgumentParser:
     Returns a parser for the command line arguments of the script.
     See README.md for more information.
     """
-    parser = ArgumentParser(prog="snippets_translate")
+    parser = ArgumentParser(prog="snippets_translate",
+                            description=HELP,
+                            formatter_class=RawDescriptionHelpFormatter)
     parser.add_argument(
         "--qt",
         action="store",
@@ -163,32 +184,39 @@ def is_valid_file(x):
     return True
 
 
-def get_snippet_ids(line: str) -> List[str]:
+def get_snippet_ids(line: str, pattern: re.Pattern) -> List[str]:
     # Extract the snippet ids for a line '//! [1] //! [2]'
     result = []
-    for m in SNIPPET_PATTERN.finditer(line):
+    for m in pattern.finditer(line):
         result.append(m.group(1))
     return result
 
 
+def overriden_snippet_lines(lines: List[str], start_id: str) -> List[str]:
+    """Wrap an overridden snippet with marker and id lines."""
+    id_string = f"//! [{start_id}]"
+    result = [OVERRIDDEN_SNIPPET, id_string]
+    result.extend(lines)
+    result.append(id_string)
+    return result
+
+
 def get_snippet_override(start_id: str, rel_path: str) -> List[str]:
-    # Check if the snippet is overridden by a local file
+    """Check if the snippet is overridden by a local file under
+       sources/pyside6/doc/snippets."""
     file_start_id = start_id.replace(' ', '_')
     override_name = f"{rel_path.stem}_{file_start_id}{rel_path.suffix}.py"
     override_path = SOURCE_PATH / rel_path.parent / override_name
-    snippet = []
-    if override_path.is_file():
-        snippet.append(OVERRIDDEN_SNIPPET)
-        id_string = f"//! [{start_id}]"
-        snippet.append(id_string)
-        snippet.extend(override_path.read_text().splitlines())
-        snippet.append(id_string)
-    return snippet
+    if not override_path.is_file():
+        return []
+    lines = override_path.read_text().splitlines()
+    return overriden_snippet_lines(lines, start_id)
 
 
-def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
-    # Extract (potentially overlapping) snippets from a C++ file indicated by //! [1]
-    snippets: List[List[str]] = []
+def _get_snippets(lines: List[str], pattern: re.Pattern) -> Dict[str, List[str]]:
+    """Helper to extract (potentially overlapping) snippets from a C++ file
+       indicated by pattern ("//! [1]") and return them as a dict by <id>."""
+    snippets: Dict[str, List[str]] = {}
     snippet: List[str]
     done_snippets : List[str] = []
 
@@ -197,19 +225,14 @@ def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
         line = lines[i]
         i += 1
 
-        start_ids = get_snippet_ids(line)
+        start_ids = get_snippet_ids(line, pattern)
         while start_ids:
             # Start of a snippet
             start_id = start_ids.pop(0)
             if start_id in done_snippets:
                 continue
             done_snippets.append(start_id)
-            snippet = get_snippet_override(start_id, rel_path)
-            if snippet:
-                snippets.append(snippet)
-                continue
-
-            snippet.append(line)  # The snippet starts with this id
+            snippet = [line]  # The snippet starts with this id
 
             # Find the end of the snippet
             j = i
@@ -221,14 +244,46 @@ def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
                 snippet.append(l)
 
                 # Check if the snippet is complete
-                if start_id in get_snippet_ids(l):
+                if start_id in get_snippet_ids(l, pattern):
                     # End of snippet
-                    snippets.append(snippet)
+                    snippets[start_id] = snippet
                     break
 
     return snippets
 
 
+def get_python_example_snippet_override(start_id: str, rel_path: str) -> List[str]:
+    """Check if the snippet is overridden by a python example snippet."""
+    key = (os.fspath(rel_path), start_id)
+    value = python_example_snippet_mapping().get(key)
+    if not value:
+        return []
+    path, id = value
+    file_lines = path.read_text().splitlines()
+    snippet_dict = _get_snippets(file_lines, PYTHON_SNIPPET_PATTERN)
+    lines = snippet_dict.get(id)
+    if not lines:
+        raise RuntimeError(f'Snippet "{id}" not found in "{os.fspath(path)}"')
+    lines = lines[1:-1]  # Strip Python snippet markers
+    return overriden_snippet_lines(lines, start_id)
+
+
+def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
+    """Extract (potentially overlapping) snippets from a C++ file indicated
+       by '//! [1]'."""
+    result = _get_snippets(lines, CPP_SNIPPET_PATTERN)
+    id_list = result.keys()
+    for snippet_id in id_list:
+        # Check file overrides and example overrides
+        snippet = get_snippet_override(snippet_id, rel_path)
+        if not snippet:
+            snippet = get_python_example_snippet_override(snippet_id, rel_path)
+        if snippet:
+            result[snippet_id] = snippet
+
+    return result.values()
+
+
 def get_license_from_file(filename):
     lines = []
     with open(filename, "r") as f:
diff --git a/tools/snippets_translate/override.py b/tools/snippets_translate/override.py
new file mode 100644
index 000000000..c9aa6d498
--- /dev/null
+++ b/tools/snippets_translate/override.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2022 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+from pathlib import Path
+
+ROOT_PATH = Path(__file__).parents[2]
+EXAMPLES_PATH = ROOT_PATH / "examples"
+
+
+_PYTHON_EXAMPLE_SNIPPET_MAPPING = {
+    ("qtbase/examples/widgets/tutorials/modelview/1_readonly/mymodel.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "1_readonly.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/2_formatting/mymodel.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "2_formatting.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_QVariant"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "2"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_a"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_b"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "3"),
+    ("qtbase/examples/widgets/tutorials/modelview/4_headers/mymodel.cpp",
+     "quoting mymodel_c"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "4_headers.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp",
+     "quoting mymodel_e"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp",
+     "quoting mymodel_f"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "2"),
+    ("qtbase/examples/widgets/tutorials/modelview/6_treeview/mainwindow.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "6_treeview.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp",
+     "quoting modelview_a"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp",
+     "quoting modelview_b"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "2")
+}
+
+
+_python_example_snippet_mapping = {}
+
+
+def python_example_snippet_mapping():
+    global _python_example_snippet_mapping
+    if not _python_example_snippet_mapping:
+        result = _PYTHON_EXAMPLE_SNIPPET_MAPPING
+        _python_example_snippet_mapping = result
+
+    return _python_example_snippet_mapping
diff --git a/tools/snippets_translate/snippets_translate.pyproject b/tools/snippets_translate/snippets_translate.pyproject
index 8016eb637..6073e9b89 100644
--- a/tools/snippets_translate/snippets_translate.pyproject
+++ b/tools/snippets_translate/snippets_translate.pyproject
@@ -1,3 +1,3 @@
 {
-    "files": ["main.py", "converter.py", "handlers.py", "tests/test_converter.py"]
+    "files": ["main.py", "converter.py", "handlers.py", "override.py", "tests/test_converter.py"]
 }