snippets_translate: Add a way of using snippets from Python

Split a helper off the snippet extraction function taking the comment
pattern and let it return a dict by id to make it possible to replace
snippets by id.

Prototypically use it for the  modelview tutorial.

Fixes: PYSIDE-1984
Task-number: PYSIDE-1952
Change-Id: I05dbc3e36825761fe2968d6507880cd6f588682d
Reviewed-by: Christian Tismer <tismer@stackless.com>
(cherry picked from commit c4a266e38fe5bdce707ad6b123fa88bb4f10dff3)
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

10 files changed, 198 insertions, 32 deletions

diff --git a/examples/widgets/tutorials/modelview/1_readonly.py b/examples/widgets/tutorials/modelview/1_readonly.py
index 96bf04a57..a31762f7d 100644
--- a/examples/widgets/tutorials/modelview/1_readonly.py
+++ b/examples/widgets/tutorials/modelview/1_readonly.py
@@ -46,6 +46,7 @@ from PySide6.QtWidgets import QApplication, QTableView
 """PySide6 port of the widgets/tutorials/modelview/1_readonly example from Qt v6.x"""
 
 
+#! [1]
 class MyModel(QAbstractTableModel):
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -62,6 +63,7 @@ class MyModel(QAbstractTableModel):
             column = index.column() + 1
             return f"Row{row}, Column{column}"
         return None
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/2_formatting.py b/examples/widgets/tutorials/modelview/2_formatting.py
index ac6c7e5dd..6748d5c93 100644
--- a/examples/widgets/tutorials/modelview/2_formatting.py
+++ b/examples/widgets/tutorials/modelview/2_formatting.py
@@ -57,6 +57,7 @@ class MyModel(QAbstractTableModel):
     def columnCount(self, parent=None):
         return 3
 
+#! [1]
     def data(self, index, role=Qt.DisplayRole):
         row = index.row()
         col = index.column()
@@ -89,6 +90,7 @@ class MyModel(QAbstractTableModel):
                 return Qt.Checked
 
         return None
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/3_changingmodel.py b/examples/widgets/tutorials/modelview/3_changingmodel.py
index aab0e04c6..4d089bbd2 100644
--- a/examples/widgets/tutorials/modelview/3_changingmodel.py
+++ b/examples/widgets/tutorials/modelview/3_changingmodel.py
@@ -47,12 +47,14 @@ from PySide6.QtWidgets import QApplication, QTableView
 
 
 class MyModel(QAbstractTableModel):
+#! [1]
     def __init__(self, parent=None):
         super().__init__(parent)
         self._timer = QTimer(self)
         self._timer.setInterval(1000)
         self._timer.timeout.connect(self.timer_hit)
         self._timer.start()
+#! [1]
 
     def rowCount(self, parent=None):
         return 2
@@ -60,20 +62,23 @@ class MyModel(QAbstractTableModel):
     def columnCount(self, parent=None):
         return 3
 
+#! [2]
     def data(self, index, role=Qt.DisplayRole):
         row = index.row()
         col = index.column()
         if role == Qt.DisplayRole and row == 0 and col == 0:
             return QTime.currentTime().toString()
         return None
+#! [2]
 
+#! [3]
     @Slot()
     def timer_hit(self):
         # we identify the top left cell
         top_left = self.createIndex(0, 0)
         # emit a signal to make the view reread identified data
         self.dataChanged.emit(top_left, top_left, [Qt.DisplayRole])
-
+#! [3]
 
 if __name__ == '__main__':
     app = QApplication(sys.argv)
diff --git a/examples/widgets/tutorials/modelview/4_headers.py b/examples/widgets/tutorials/modelview/4_headers.py
index 0a2f05feb..0328812e9 100644
--- a/examples/widgets/tutorials/modelview/4_headers.py
+++ b/examples/widgets/tutorials/modelview/4_headers.py
@@ -63,10 +63,12 @@ class MyModel(QAbstractTableModel):
             return f"Row{row}, Column{column}"
         return None
 
+#! [1]
     def headerData(self, section, orientation, role):
         if role == Qt.DisplayRole and orientation == Qt.Horizontal:
             return ["first", "second", "third"][section]
         return None
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/5_edit.py b/examples/widgets/tutorials/modelview/5_edit.py
index b43876cbc..ac62b4524 100644
--- a/examples/widgets/tutorials/modelview/5_edit.py
+++ b/examples/widgets/tutorials/modelview/5_edit.py
@@ -70,6 +70,7 @@ class MyModel(QAbstractTableModel):
             return self._grid_data[index.row()][index.column()]
         return None
 
+#! [1]
     def setData(self, index, value, role):
         if role != Qt.EditRole or not self.checkIndex(index):
             return False
@@ -79,9 +80,12 @@ class MyModel(QAbstractTableModel):
         result = " ".join(chain(*self._grid_data))
         self.editCompleted.emit(result)
         return True
+#! [1]
 
+#! [2]
     def flags(self, index):
         return Qt.ItemIsEditable | super().flags(index)
+#! [2]
 
 
 class MainWindow(QMainWindow):
diff --git a/examples/widgets/tutorials/modelview/6_treeview.py b/examples/widgets/tutorials/modelview/6_treeview.py
index 5161a9550..1338e1476 100644
--- a/examples/widgets/tutorials/modelview/6_treeview.py
+++ b/examples/widgets/tutorials/modelview/6_treeview.py
@@ -46,7 +46,7 @@ from PySide6.QtWidgets import QApplication, QMainWindow, QTreeView
 
 """PySide6 port of the widgets/tutorials/modelview/6_treeview example from Qt v6.x"""
 
-
+#! [1]
 class MainWindow(QMainWindow):
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -69,6 +69,7 @@ class MainWindow(QMainWindow):
     def prepare_row(self, first, second, third):
         return [QStandardItem(first), QStandardItem(second),
                 QStandardItem(third)]
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/7_selections.py b/examples/widgets/tutorials/modelview/7_selections.py
index 094d2f4bf..d2f2814a9 100644
--- a/examples/widgets/tutorials/modelview/7_selections.py
+++ b/examples/widgets/tutorials/modelview/7_selections.py
@@ -46,7 +46,7 @@ from PySide6.QtWidgets import QApplication, QMainWindow, QTreeView
 
 """PySide6 port of the widgets/tutorials/modelview/7_selections example from Qt v6.x"""
 
-
+#! [1]
 class MainWindow(QMainWindow):
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -82,7 +82,9 @@ class MainWindow(QMainWindow):
         # selection changes shall trigger a slot
         selection_model = self._tree_view.selectionModel()
         selection_model.selectionChanged.connect(self.selection_changed_slot)
+#! [1]
 
+#! [2]
     @Slot(QItemSelection, QItemSelection)
     def selection_changed_slot(self, new_selection, old_selection):
         # get the text of the selected item
@@ -95,6 +97,7 @@ class MainWindow(QMainWindow):
             seek_root = seek_root.parent()
             hierarchy_level += 1
         self.setWindowTitle(f"{selected_text}, Level {hierarchy_level}")
+#! [2]
 
 
 if __name__ == '__main__':
diff --git a/tools/snippets_translate/main.py b/tools/snippets_translate/main.py
index 37f6e3cb9..d4b8e35b4 100644
--- a/tools/snippets_translate/main.py
+++ b/tools/snippets_translate/main.py
@@ -41,14 +41,28 @@ import logging
 import os
 import re
 import sys
-from argparse import ArgumentParser, Namespace
+from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
 from enum import Enum
 from pathlib import Path
 from textwrap import dedent
-from typing import List
+from typing import Dict, List
 
+from override import python_example_snippet_mapping
 from converter import snippet_translate
 
+HELP = """Converts Qt C++ code snippets to Python snippets.
+
+Ways to override Snippets:
+
+1) Complete snippets from local files:
+   To replace snippet "[1]" of "foo/bar.cpp", create a file
+   "sources/pyside6/doc/snippets/foo/bar_1.cpp.py" .
+2) Snippets extracted from Python examples:
+   To use snippets from Python examples, add markers ("#! [id]") to it
+   and an entry to _PYTHON_EXAMPLE_SNIPPET_MAPPING.
+"""
+
+
 # Logger configuration
 try:
     from rich.logging import RichHandler
@@ -73,11 +87,16 @@ log = logging.getLogger("snippets_translate")
 # Filter and paths configuration
 SKIP_END = (".pro", ".pri", ".cmake", ".qdoc", ".yaml", ".frag", ".qsb", ".vert", "CMakeLists.txt")
 SKIP_BEGIN = ("changes-", ".")
-SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]")
+CPP_SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]")
+PYTHON_SNIPPET_PATTERN = re.compile(r"#! ?\[([^]]+)\]")
+
+ROOT_PATH = Path(__file__).parents[2]
+SOURCE_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "snippets"
+
 
-SOURCE_PATH = Path(__file__).parents[2] / "sources" / "pyside6" / "doc" / "snippets"
 OVERRIDDEN_SNIPPET = "# OVERRIDDEN_SNIPPET"
 
+
 class FileStatus(Enum):
     Exists = 0
     New = 1
@@ -88,7 +107,9 @@ def get_parser() -> ArgumentParser:
     Returns a parser for the command line arguments of the script.
     See README.md for more information.
     """
-    parser = ArgumentParser(prog="snippets_translate")
+    parser = ArgumentParser(prog="snippets_translate",
+                            description=HELP,
+                            formatter_class=RawDescriptionHelpFormatter)
     parser.add_argument(
         "--qt",
         action="store",
@@ -199,32 +220,39 @@ def is_valid_file(x):
     return True
 
 
-def get_snippet_ids(line: str) -> List[str]:
+def get_snippet_ids(line: str, pattern: re.Pattern) -> List[str]:
     # Extract the snippet ids for a line '//! [1] //! [2]'
     result = []
-    for m in SNIPPET_PATTERN.finditer(line):
+    for m in pattern.finditer(line):
         result.append(m.group(1))
     return result
 
 
+def overriden_snippet_lines(lines: List[str], start_id: str) -> List[str]:
+    """Wrap an overridden snippet with marker and id lines."""
+    id_string = f"//! [{start_id}]"
+    result = [OVERRIDDEN_SNIPPET, id_string]
+    result.extend(lines)
+    result.append(id_string)
+    return result
+
+
 def get_snippet_override(start_id: str, rel_path: str) -> List[str]:
-    # Check if the snippet is overridden by a local file
+    """Check if the snippet is overridden by a local file under
+       sources/pyside6/doc/snippets."""
     file_start_id = start_id.replace(' ', '_')
     override_name = f"{rel_path.stem}_{file_start_id}{rel_path.suffix}.py"
     override_path = SOURCE_PATH / rel_path.parent / override_name
-    snippet = []
-    if override_path.is_file():
-        snippet.append(OVERRIDDEN_SNIPPET)
-        id_string = f"//! [{start_id}]"
-        snippet.append(id_string)
-        snippet.extend(override_path.read_text().splitlines())
-        snippet.append(id_string)
-    return snippet
+    if not override_path.is_file():
+        return []
+    lines = override_path.read_text().splitlines()
+    return overriden_snippet_lines(lines, start_id)
 
 
-def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
-    # Extract (potentially overlapping) snippets from a C++ file indicated by //! [1]
-    snippets: List[List[str]] = []
+def _get_snippets(lines: List[str], pattern: re.Pattern) -> Dict[str, List[str]]:
+    """Helper to extract (potentially overlapping) snippets from a C++ file
+       indicated by pattern ("//! [1]") and return them as a dict by <id>."""
+    snippets: Dict[str, List[str]] = {}
     snippet: List[str]
     done_snippets : List[str] = []
 
@@ -233,19 +261,14 @@ def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
         line = lines[i]
         i += 1
 
-        start_ids = get_snippet_ids(line)
+        start_ids = get_snippet_ids(line, pattern)
         while start_ids:
             # Start of a snippet
             start_id = start_ids.pop(0)
             if start_id in done_snippets:
                 continue
             done_snippets.append(start_id)
-            snippet = get_snippet_override(start_id, rel_path)
-            if snippet:
-                snippets.append(snippet)
-                continue
-
-            snippet.append(line)  # The snippet starts with this id
+            snippet = [line]  # The snippet starts with this id
 
             # Find the end of the snippet
             j = i
@@ -257,14 +280,46 @@ def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
                 snippet.append(l)
 
                 # Check if the snippet is complete
-                if start_id in get_snippet_ids(l):
+                if start_id in get_snippet_ids(l, pattern):
                     # End of snippet
-                    snippets.append(snippet)
+                    snippets[start_id] = snippet
                     break
 
     return snippets
 
 
+def get_python_example_snippet_override(start_id: str, rel_path: str) -> List[str]:
+    """Check if the snippet is overridden by a python example snippet."""
+    key = (os.fspath(rel_path), start_id)
+    value = python_example_snippet_mapping().get(key)
+    if not value:
+        return []
+    path, id = value
+    file_lines = path.read_text().splitlines()
+    snippet_dict = _get_snippets(file_lines, PYTHON_SNIPPET_PATTERN)
+    lines = snippet_dict.get(id)
+    if not lines:
+        raise RuntimeError(f'Snippet "{id}" not found in "{os.fspath(path)}"')
+    lines = lines[1:-1]  # Strip Python snippet markers
+    return overriden_snippet_lines(lines, start_id)
+
+
+def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
+    """Extract (potentially overlapping) snippets from a C++ file indicated
+       by '//! [1]'."""
+    result = _get_snippets(lines, CPP_SNIPPET_PATTERN)
+    id_list = result.keys()
+    for snippet_id in id_list:
+        # Check file overrides and example overrides
+        snippet = get_snippet_override(snippet_id, rel_path)
+        if not snippet:
+            snippet = get_python_example_snippet_override(snippet_id, rel_path)
+        if snippet:
+            result[snippet_id] = snippet
+
+    return result.values()
+
+
 def get_license_from_file(filename):
     lines = []
     with open(filename, "r") as f:
diff --git a/tools/snippets_translate/override.py b/tools/snippets_translate/override.py
new file mode 100644
index 000000000..f0f76fd2b
--- /dev/null
+++ b/tools/snippets_translate/override.py
@@ -0,0 +1,92 @@
+#############################################################################
+##
+## Copyright (C) 2022 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+from pathlib import Path
+
+ROOT_PATH = Path(__file__).parents[2]
+EXAMPLES_PATH = ROOT_PATH / "examples"
+
+
+_PYTHON_EXAMPLE_SNIPPET_MAPPING = {
+    ("qtbase/examples/widgets/tutorials/modelview/1_readonly/mymodel.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "1_readonly.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/2_formatting/mymodel.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "2_formatting.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_QVariant"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "2"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_a"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_b"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "3"),
+    ("qtbase/examples/widgets/tutorials/modelview/4_headers/mymodel.cpp",
+     "quoting mymodel_c"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "4_headers.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp",
+     "quoting mymodel_e"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp",
+     "quoting mymodel_f"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "2"),
+    ("qtbase/examples/widgets/tutorials/modelview/6_treeview/mainwindow.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "6_treeview.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp",
+     "quoting modelview_a"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp",
+     "quoting modelview_b"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "2")
+}
+
+
+_python_example_snippet_mapping = {}
+
+
+def python_example_snippet_mapping():
+    global _python_example_snippet_mapping
+    if not _python_example_snippet_mapping:
+        result = _PYTHON_EXAMPLE_SNIPPET_MAPPING
+        _python_example_snippet_mapping = result
+
+    return _python_example_snippet_mapping
diff --git a/tools/snippets_translate/snippets_translate.pyproject b/tools/snippets_translate/snippets_translate.pyproject
index 8016eb637..6073e9b89 100644
--- a/tools/snippets_translate/snippets_translate.pyproject
+++ b/tools/snippets_translate/snippets_translate.pyproject
@@ -1,3 +1,3 @@
 {
-    "files": ["main.py", "converter.py", "handlers.py", "tests/test_converter.py"]
+    "files": ["main.py", "converter.py", "handlers.py", "override.py", "tests/test_converter.py"]
 }