Merge branch 'release/1.7'1.7

12 files changed, 240 insertions, 106 deletions

diff --git a/cli.py b/cli.py
index 39ca71a..feab43a 100755
--- a/cli.py
+++ b/cli.py
@@ -186,5 +186,12 @@ def docs_serve():
     server.serve(root='docs/_build/html', open_url=True)
 
 
+@cli.command()
+def clean():
+    Path('build').rmtree_p()
+    Path('dist').rmtree_p()
+    Path('qface.egg-info').rmtree_p()
+
+
 if __name__ == '__main__':
     cli()
diff --git a/docs/builtin.rst b/docs/builtin.rst
index b39746c..a24f424 100644
--- a/docs/builtin.rst
+++ b/docs/builtin.rst
@@ -1,17 +1,31 @@
-Builtin Generators
+Generator Examples
 ==================
 
-QFace contains several built in code generators. Their purpose is merely to showcase how to write a code generator
-with QFace. They are working and complete examples of a general purpose generators.
+QFace does provide soem real world generators which are hosted as separated projects. Their purpose is merely to showcase how to write a code generator with QFace. They are working and complete examples of a general purpose generators.
 
-* The QtCPP generator generates a Qt C++ plugin with a QML API ready to be used in your project.
-* The QtQml generator generates a QML only API which ready to be used.
+`qface-qtcpp`_
 
-From the QML user interface perspective both provide the same API and are interchangeable.
+    The QtCPP generator generates a Qt C++ plugin with a QML API ready to be used in your project.
 
+    Hosted at: https://github.com/Pelagicore/qface-qtcpp
 
-.. toctree::
-   :maxdepth: 1
+`qface-qtqml`_
 
-   qtcpp
-   qtqml
+    The QtQml generator generates a QML only API which ready to be used.
+
+    Hosted at: https://github.com/Pelagicore/qface-qtqml
+
+`qface-qtro`_
+
+    The RO (RemoteObjects) generator generates a client and server project using the Qt5 QtRemoteObejcts library
+
+    Hosted at: https://github.com/Pelagicore/qface-qtro
+
+
+From the QML user interface perspective the QtCPP and QtQML generators bth provide the same API and are interchangeable.
+
+
+
+.. _qface-qtcpp: https://github.com/Pelagicore/qface-qtcpp
+.. _qface-qtqml: https://github.com/Pelagicore/qface-qtqml
+.. _qface-qtro: https://github.com/Pelagicore/qface-qtro
\ No newline at end of file
diff --git a/docs/extending.rst b/docs/extending.rst
index df751c2..010c810 100644
--- a/docs/extending.rst
+++ b/docs/extending.rst
@@ -53,3 +53,73 @@ structs and enums.
 
 The template code iterates over the domain objects and generates text using a
 mixture of output blocks ``{{}}`` and control blocks ``{%%}``.
+
+
+Rule Base Generation
+====================
+
+The `RuleGenerator` allows you to extract the documentation rules into an external yaml file. This makes the python script more compact.
+
+
+.. code-block:: python
+
+    from qface.generator import FileSystem, RuleGenerator
+    from path import Path
+
+    here = Path(__file__).dirname()
+
+    def generate(input, output):
+        # parse the interface files
+        system = FileSystem.parse(input)
+        # setup the generator
+        generator = RuleGenerator(search_path=here/'templates', destination=output)
+        generator.process_rules(here/'docs.yaml', system)
+
+The rules document is divided into several targets. Each target can have an own destination. A target is typical for exampe and app, client, server. Each target can have rules for the different symbols (system, module, interface, struct, enum). An each rule finally consists of a destination modifier, additional context and a documents collection.
+
+.. code-block:: python
+
+    <target>:
+        <symbol>:
+            context: {}
+            destination: ''
+            documents:
+                <target>:<source>
+
+* ``<target>`` is a name of the current target (e.g. client, server, plugin)
+* ``<symbol>`` must be either system, module, interface, struct or enum
+
+
+Here is an example (``docs.yaml``)
+
+.. code-block:: yaml
+
+    global:
+        destination: '{{dst}}'
+        system:
+            documents:
+                '{{project}}.pro': 'project.pro'
+                '.qmake.conf': 'qmake.conf'
+                'CMakeLists.txt': 'CMakeLists.txt'
+    plugin:
+        destination: '{{dst}}/plugin'
+        module:
+            context: {'module_name': '{{module|identifier}}'}
+            documents:
+                '{{module_name}}.pro': 'plugin/plugin.pro'
+                'CMakeLists.txt': 'plugin/CMakeLists.txt'
+                'plugin.cpp': 'plugin/plugin.cpp'
+                'plugin.h': 'plugin/plugin.h'
+                'qmldir': 'plugin/qmldir'
+        interface:
+            documents:
+                '{{interface|lower}}.h': 'plugin/interface.h'
+                '{{interface|lower}}.cpp': 'plugin/interface.cpp'
+        struct:
+            documents:
+                '{{struct|lower}}.h': 'plugin/struct.h'
+                '{{struct|lower}}.cpp': 'plugin/struct.cpp'
+
+
+The rule generator adds the ``dst``, ``project`` as also the corresponding symbols to the context automatically. On each level you are able to change the destination or update the context.
+
diff --git a/docs/qface_concept.jpg b/docs/qface_concept.jpg
new file mode 100644
index 0000000..0fde3a2
--- /dev/null
+++ b/docs/qface_concept.jpg
diff --git a/docs/qface_concept.png b/docs/qface_concept.png
deleted file mode 100644
index eb31dbd..0000000
--- a/docs/qface_concept.png
+++ /dev/null
diff --git a/docs/usage.rst b/docs/usage.rst
index af0e06b..7105bfa 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -7,64 +7,32 @@ Concept
 
 QFace requires one or more IDL files as input file and a generator to produce output files. The IDL files are named QFace interface documents.
 
-.. image:: qface_concept.png
+.. figure:: qface_concept.jpg
 
-There are several ways to call the generator.
-
-
-Invocation
-==========
-
-Direct Invocation
------------------
-
-You can call the generator directly by using the provided script. All generators should at minimum expect a series of inputs and one output path. This is normally recommended for production.
-
-.. code-block:: sh
-
-    ./csv.py src dst
-
-Via qface invokation
---------------------
-
-You can invoke your generator using the qface helper script. This allows you also to use some specific developer support. It is recommended way during generator development.
-
-To use an existing generator just provide the path to the generator script.
-
-.. code-block:: sh
-
-    qface generate --generator ./csvgen.py input output
-
-
-To use live reloading on changes just use the reload option:
-
-
-.. code-block:: sh
-
-    qface generate --generator ./csvgen.py input output --reload
-
-This will observe the generator folder and the input folder for changes and re-run the generator.
-
-Configuration Invokation
-------------------------
-
-You can also create a YAML configuration file (for example csv.yaml):
+To use qface you need to write your own generator. A generatopr is a small python script which reads the qface document and write code using a generator.
 
+.. code-block:: python
 
-.. code-block:: yaml
+    # gen.py
+    from qface.generator import FileSystem, Generator
 
-    generator: ./csvgen.py
-    input: input
-    output: output
-    reload: false
+    def generate(input, output):
+        # parse the interface files
+        system = FileSystem.parse(input)
+        # setup the generator
+        generator = Generator(search_path='templates')
+        # create a context object
+        ctx = {'output': output, 'system': system}
+        # apply the context on the template and write the output to file
+        generator.write('{{output}}/modules.csv', 'modules.csv', ctx)
 
+    # call the generation function
+    generate('sample.qface', 'out')
 
-And then call the client with:
 
 .. code-block:: sh
 
-    qface generate --config csv.yaml
-
+    python3 gen.py
 
 
 Code Generation Principle
@@ -99,4 +67,4 @@ This script reads the input directory returns a system object form the domain mo
         {% endfor -%}
     {% endfor %}
 
-The template iterates over the domain objects and generates text which is written into a file. Using the generator write method ``generator.write(path, template, context)`` the output file path can also be specified using the template syntax . 
+The template iterates over the domain objects and generates text which is written into a file. Using the generator write method ``generator.write(path, template, context)`` the output file path can also be specified using the template syntax .
diff --git a/qface/__about__.py b/qface/__about__.py
index 0d6a726..4fbd624 100644
--- a/qface/__about__.py
+++ b/qface/__about__.py
@@ -9,7 +9,7 @@ except NameError:
 __title__ = "qface"
 __summary__ = "A generator framework based on a common modern IDL"
 __url__ = "https://pelagicore.github.io/qface/"
-__version__ = "1.6"
+__version__ = "1.7"
 __author__ = "JRyannel"
 __author_email__ = "qface-generator@googlegroups.com"
 __copyright__ = "2017 Pelagicore"
diff --git a/qface/filters.py b/qface/filters.py
index c5ecab8..de424a4 100644
--- a/qface/filters.py
+++ b/qface/filters.py
@@ -12,10 +12,15 @@ def jsonify(symbol):
     return json.dumps(symbol, indent='  ')
 
 
-def upper_first(symbol):
+def upper_first(s):
     """ uppercase first letter """
-    name = str(symbol)
-    return name[0].upper() + name[1:]
+    s = str(s)
+    return s[0].upper() + s[1:]
+
+
+def lower_first(s):
+    s = str(s)
+    return s[0].lower() + s[1:]
 
 
 def hash(symbol, hash_type='sha1'):
@@ -28,3 +33,14 @@ def hash(symbol, hash_type='sha1'):
 def path(symbol):
     """ replaces '.' with '/' """
     return str(symbol).replace('.', '/')
+
+
+filters = {
+    'jsonify': jsonify,
+    'upper_first': upper_first,
+    'lower_first': lower_first,
+    'upperfirst': upper_first,
+    'lowerfirst': lower_first,
+    'hash': hash,
+    'path': path,
+}
diff --git a/qface/generator.py b/qface/generator.py
index c2e018b..572cedf 100644
--- a/qface/generator.py
+++ b/qface/generator.py
@@ -20,6 +20,7 @@ from .idl.parser.TListener import TListener
 from .idl.domain import System
 from .idl.listener import DomainListener
 from .utils import merge
+from .filters import filters
 
 
 try:
@@ -34,17 +35,6 @@ logger = logging.getLogger(__name__)
 Provides an API for accessing the file system and controlling the generator
 """
 
-
-def upper_first_filter(s):
-    s = str(s)
-    return s[0].upper() + s[1:]
-
-
-def lower_first_filter(s):
-    s = str(s)
-    return s[0].lower() + s[1:]
-
-
 class ReportingErrorListener(ErrorListener.ErrorListener):
     def __init__(self, document):
         self.document = document
@@ -69,7 +59,7 @@ class Generator(object):
     strict = False
     """ enables strict code generation """
 
-    def __init__(self, search_path: str):
+    def __init__(self, search_path: str, context: dict={}):
         loader = ChoiceLoader([
             FileSystemLoader(search_path),
             PackageLoader('qface')
@@ -79,9 +69,9 @@ class Generator(object):
             trim_blocks=True,
             lstrip_blocks=True
         )
-        self.env.filters['upperfirst'] = upper_first_filter
-        self.env.filters['lowerfirst'] = lower_first_filter
+        self.env.filters.update(filters)
         self._destination = Path()
+        self.context = context
 
     @property
     def destination(self):
@@ -90,7 +80,16 @@ class Generator(object):
 
     @destination.setter
     def destination(self, dst: str):
-        self._destination = Path(dst)
+        if dst:
+            self._destination = Path(self.apply(dst, self.context))
+
+    @property
+    def filters(self):
+        return self.env.filters
+
+    @filters.setter
+    def filters(self, filters):
+        self.env.filters.update(filters)
 
     def get_template(self, name: str):
         """Retrieves a single template file from the template loader"""
@@ -106,10 +105,12 @@ class Generator(object):
         """Return the rendered text of a template instance"""
         return self.env.from_string(template).render(context)
 
-    def write(self, file_path: Path, template: str, context: dict, preserve: bool = False):
+    def write(self, file_path: Path, template: str, context: dict={}, preserve: bool = False):
         """Using a template file name it renders a template
            into a file given a context
         """
+        if not context:
+            context = self.context
         error = False
         try:
             self._write(file_path, template, context, preserve)
@@ -153,6 +154,49 @@ class Generator(object):
         self.env.filters[name] = callback
 
 
+class RuleGenerator(Generator):
+    """Generates documents based on a rule YAML document"""
+    def __init__(self, search_path: str, destination: Path, context: dict= {}):
+        super().__init__(search_path, context)
+        self.context.update({
+            'dst': destination,
+            'project': Path(destination).name,
+        })
+        self.destination = '{{dst}}'
+
+    def process_rules(self, document: Path, system: System):
+        """writes the templates read from the rules document"""
+        self.context.update({'system': system})
+        rules = FileSystem.load_yaml(document, required=True)
+        for name, target in rules.items():
+            click.secho('process target: {0}'.format(name), fg='green')
+            self._process_target(target, system)
+
+    def _process_target(self, rules: dict, system: System):
+        """ process a set of rules for a target """
+        self.context.update(rules.get('context', {}))
+        self.destination = rules.get('destination', '{{dst}}')
+        self._process_rule(rules.get('system', None), {'system': system})
+        for module in system.modules:
+            self._process_rule(rules.get('module', None), {'module': module})
+            for interface in module.interfaces:
+                self._process_rule(rules.get('interface', None), {'interface': interface})
+            for struct in module.structs:
+                self._process_rule(rules.get('struct', None), {'struct': struct})
+            for enum in module.enums:
+                self._process_rule(rules.get('enum', None), {'enum': enum})
+
+    def _process_rule(self, rule: dict, context: dict):
+        """ process a single rule """
+        if not rule:
+            return
+        self.context.update(context)
+        self.context.update(rule.get('context', {}))
+        self.destination = rule.get('destination', None)
+        for target, source in rule.get('documents', {}).items():
+            self.write(target, source)
+
+
 class FileSystem(object):
     """QFace helper functions to work with the file system"""
     strict = False
@@ -172,7 +216,6 @@ class FileSystem(object):
         if error and FileSystem.strict:
             sys.exit(-1)
 
-
     @staticmethod
     def _parse_document(document: Path, system: System = None):
         """Parses a document and returns the resulting domain system
@@ -205,13 +248,7 @@ class FileSystem(object):
         """Read a YAML document and for each root symbol identifier
         updates the tag information of that symbol
         """
-        if not document.exists():
-            return
-        meta = {}
-        try:
-            meta = yaml.load(document.text(), Loader=Loader)
-        except yaml.YAMLError as exc:
-            click.secho(exc, fg='red')
+        meta = FileSystem.load_yaml(document)
         click.secho('merge tags from {0}'.format(document), fg='blue')
         for identifier, data in meta.items():
             symbol = system.lookup(identifier)
@@ -252,3 +289,16 @@ class FileSystem(object):
         if use_cache:
             cache[identifier] = system
         return system
+
+    @staticmethod
+    def load_yaml(document: Path, required=False):
+        document = Path(document)
+        if not document.exists():
+            if required:
+                click.secho('yaml document does not exists: {0}'.format(document), fg='red')
+            return {}
+        try:
+            return yaml.load(document.text(), Loader=Loader)
+        except yaml.YAMLError as exc:
+            click.secho(exc, fg='red')
+        return {}
diff --git a/qface/helper/qtcpp.py b/qface/helper/qtcpp.py
index db959b4..5561fe5 100644
--- a/qface/helper/qtcpp.py
+++ b/qface/helper/qtcpp.py
@@ -3,10 +3,7 @@ Provides helper functionality specificially for Qt C++/QML code generators
 """
 import qface.idl.domain as domain
 from jinja2 import environmentfilter
-
-def upper_first(s):
-    s = str(s)
-    return s[0].upper() + s[1:]
+from ..filters import upper_first
 
 
 class Filters(object):
@@ -126,6 +123,11 @@ class Filters(object):
         return 'using namespace {0};'.format(id)
 
     @staticmethod
+    def ns(symbol):
+        '''generates a namespace x::y::z statement from a symbol'''
+        return '::'.join(symbol.module.name_parts)
+
+    @staticmethod
     def signalName(s):
         if isinstance(s, domain.Property):
             return '{0}Changed'.format(s)
@@ -178,11 +180,23 @@ class Filters(object):
         return str(s).lower().replace('.', '_')
 
     @staticmethod
-    def upper_first(s):
-        s = str(s)
-        return s[0].upper() + s[1:]
-
-    @staticmethod
     def path(s):
         return str(s).replace('.', '/')
 
+    @staticmethod
+    def get_filters():
+        return {
+            'defaultValue': Filters.defaultValue,
+            'returnType': Filters.returnType,
+            'parameterType': Filters.parameterType,
+            'open_ns': Filters.open_ns,
+            'close_ns': Filters.close_ns,
+            'using_ns': Filters.using_ns,
+            'ns': Filters.ns,
+            'signalName': Filters.signalName,
+            'parameters': Filters.parameters,
+            'signature': Filters.signature,
+            'identifier': Filters.identifier,
+            'path': Filters.path,
+            'className': Filters.className,
+        }
diff --git a/qface/idl/domain.py b/qface/idl/domain.py
index a2e3972..defdca3 100644
--- a/qface/idl/domain.py
+++ b/qface/idl/domain.py
@@ -79,7 +79,7 @@ class System(object):
         return (module_name, type_name, fragment_name)
 
     def toJson(self):
-        o = {}
+        o = OrderedDict()
         o['modules'] = [o.toJson() for o in self.modules]
         return o
 
@@ -109,7 +109,7 @@ class NamedElement(object):
             return '{0}.{1}'.format(self.module.name, self.name)
 
     def toJson(self):
-        o = {}
+        o = OrderedDict()
         if self.name:
             o['name'] = self.name
         return o
@@ -197,7 +197,7 @@ class TypeSymbol(NamedElement):
         return (self.is_primitive and self.name) \
             or (self.is_complex and self.name) \
             or (self.is_list and self.nested) \
-            or (self.is_model and self.nested) \
+            or (self.is_model and self.nested)
 
     @property
     def is_bool(self):
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 6adfe04..f6bcedb 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -204,11 +204,6 @@ def test_parser_exceptions():
     path = inputPath / 'org.example.failing.qface'
     system = FileSystem.parse_document(path)
 
-    try:
-        system = FileSystem.parse_document('not-exists')
-    except SystemExit as e:
-        pass
-    else:
-        pytest.fail('should not ome here')
+    system = FileSystem.parse_document('not-exists')