Add support for parsing dotted names into tags

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
                 python-version: '3.x'
         -   name: "Main Script"
             run: |
-                EXTRA_INSTALL="pymbolic"
+                EXTRA_INSTALL="pymbolic lark-parser"
                 curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-pylint.sh
                 . ./prepare-and-run-pylint.sh "$(basename $GITHUB_REPOSITORY)" test/test_*.py
 
@@ -74,7 +74,7 @@ jobs:
                 # AK, 2020-12-13
                 rm pytools/log.py
 
-                EXTRA_INSTALL="numpy"
+                EXTRA_INSTALL="numpy lark-parser"
                 curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/build-and-test-py-project.sh
                 . ./build-and-test-py-project.sh
 
@@ -106,7 +106,7 @@ jobs:
                 python-version: '3.x'
         -   name: "Main Script"
             run: |
-                EXTRA_INSTALL="numpy"
+                EXTRA_INSTALL="numpy lark-parser"
                 curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/ci-support.sh
                 . ci-support.sh
                 build_py_project_in_venv

.gitlab-ci.yml

@@ -7,7 +7,7 @@ Pytest:
     # AK, 2020-12-13
     rm pytools/log.py
 
-    export EXTRA_INSTALL="numpy"
+    export EXTRA_INSTALL="numpy lark-parser"
     curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/build-and-test-py-project.sh
     . ./build-and-test-py-project.sh
   tags:
@@ -50,7 +50,7 @@ Mypy:
 
 Pylint:
   script:
-  - EXTRA_INSTALL="pymbolic"
+  - EXTRA_INSTALL="pymbolic lark-parser"
   - py_version=3
   - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-pylint.sh
   - . ./prepare-and-run-pylint.sh "$CI_PROJECT_NAME" test/test_*.py
@@ -61,7 +61,7 @@ Pylint:
 
 Documentation:
   script:
-  - EXTRA_INSTALL="numpy"
+  - EXTRA_INSTALL="numpy lark-parser"
   - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/build-docs.sh
   - ". ./build-docs.sh"
   tags:

MANIFEST.in

@@ -3,3 +3,5 @@ include LICENSE
 include doc/*rst
 include doc/Makefile
 include doc/conf.py
+
+include pytools/*.lark

pytools/tag.py

@@ -1,5 +1,7 @@
 from dataclasses import dataclass
 from typing import Tuple, Any, FrozenSet, Union, Iterable, TypeVar
+import importlib
+
 from pytools import memoize
 
 __copyright__ = """
@@ -40,6 +42,8 @@
 .. autoclass:: Taggable
 .. autoclass:: Tag
 .. autoclass:: UniqueTag
+.. autoclass:: ToPythonObjectMapper
+.. autofunction:: parse_tag
 
 Supporting Functionality
 ------------------------
@@ -268,4 +272,127 @@ def without_tags(self: T_co,
 
 # }}}
 
+
+# {{{ parse
+
+class CallParams:
+    """
+    Intermediate data structure for :class:`ToPythonObjectMapper`.
+    """
+    def __init__(self, args, kwargs):
+        self.args = args
+        self.kwargs = kwargs
+
+
+class ToPythonObjectMapper:
+    """
+    Map a parsed tree to pythonic objects.
+    """
+    def __init__(self, shortcuts, caller_globals):
+        super().__init__()
+        self.shortcuts = shortcuts
+        self.caller_globals = caller_globals
+
+    def map_tag_from_python_class(self, cls, params):
+        try:
+            return cls(*params.args, **params.kwargs)
+        except TypeError as e:
+            raise TypeError(f"Error while instantiating '{cls.__name__}': {e}")
+
+    def map_empty_args_params(self):
+        return CallParams((), {})
+
+    def map_args_only_params(self, args):
+        return CallParams(args, {})
+
+    def map_kwargs_only_params(self, kwargs):
+        return CallParams((), kwargs)
+
+    def map_args_kwargs_params(self, args, kwargs):
+        return CallParams(args, kwargs)
+
+    def map_name(self, tok):
+        return tok.value
+
+    def map_top_level_module(self, modulename):
+        try:
+            return self.caller_globals[modulename]
+        except KeyError:
+            return importlib.import_module(modulename)
+
+    def map_nested_module(self, module, child):
+        return getattr(module, child)
+
+    def map_tag_class(self, module, classname):
+        return getattr(module, classname)
+
+    def map_string(self, text):
+        return str(text)
+
+    def map_int(self, text):
+        return int(text)
+
+    def map_singleton_args(self, arg):
+        return (arg,)
+
+    def map_args(self, args, arg):
+        return args + (arg,)
+
+    def map_kwarg(self, name, arg):
+        return {name: arg}
+
+    def map_kwargs(self, kwargs, kwarg):
+        assert len(kwarg) == 1
+        (key, val), = kwarg.items()
+        if key in kwargs:
+            raise ValueError(f"keyword argument '{key}' repeated")
+
+        updated_kwargs = kwargs.copy()
+        updated_kwargs[key] = val
+        return updated_kwargs
+
+    def map_tag_from_shortcut(self, name):
+        name = name[1:]  # remove the starting "."
+        return self.shortcuts[name]
+
+
+@memoize
+def construct_parser():
+    from lark import Lark
+    return Lark.open("tags.lark", rel_to=__file__, parser="lalr", start="tag",
+            cache=True)
+
+
+def parse_tag(tag_text, shortcuts={}):
+    """
+    Parses a :class:`Tag` from a provided dotted name.
+    """
+    import inspect
+    from lark import Transformer
+
+    class ToPythonObjectMapperMixin(ToPythonObjectMapper, Transformer):
+        def _call_userfunc(self, tree, new_children=None):
+            """
+            Flattens the arguments before feeding to the mapper methods.
+            """
+            # Assumes tree is already transformed
+            children = new_children if new_children is not None else tree.children
+            try:
+                f = getattr(self, tree.data)
+            except AttributeError:
+                return self.__default__(tree.data, children, tree.meta)
+            else:
+                # flatten the args
+                return f(*children)
+
+    parser = construct_parser()
+
+    caller_globals = inspect.currentframe().f_back.f_globals
+    tag = ToPythonObjectMapperMixin(shortcuts, caller_globals).transform(
+            parser.parse(tag_text))
+
+    return tag
+
+# }}}
+
 # vim: foldmethod=marker

pytools/tags.lark

@@ -0,0 +1,35 @@
+tag: tag_class "(" params ")" -> map_tag_from_python_class
+   | SHORTCUT                 -> map_tag_from_shortcut
+
+params:                       -> map_empty_args_params
+      | args                  -> map_args_only_params
+      | kwargs                -> map_kwargs_only_params
+      | args "," kwargs       -> map_args_kwargs_params
+
+?kwargs: kwarg
+       | kwargs "," kwarg     -> map_kwargs
+
+args: arg                     -> map_singleton_args
+    | args "," arg            -> map_args
+
+kwarg: name "=" arg           -> map_kwarg
+
+?arg: tag
+    | INT                     -> map_int
+    | ESCAPED_STRING          -> map_string
+
+tag_class: module "." name    -> map_tag_class
+
+module: name                  -> map_top_level_module
+      | module "." name       -> map_nested_module
+
+name: CNAME                   -> map_name
+SHORTCUT: "." ("_"|LETTER) ("_"|LETTER|DIGIT|".")*
+
+%import common.INT
+%import common.ESCAPED_STRING
+%import common.DIGIT
+%import common.LETTER
+%import common.CNAME
+%import common.WS
+%ignore WS

setup.py

@@ -41,9 +41,13 @@
           "decorator>=3.2.0",
           "appdirs>=1.4.0",
           "numpy>=1.6.0",
-          "dataclasses>=0.7;python_version<='3.6'"
+          "dataclasses>=0.7;python_version<='3.6'",
           ],
 
+      extras_require={
+          "tag_parsing": ["lark-parser"],
+          },
+
       package_data={"pytools": ["py.typed"]},
 
       author="Andreas Kloeckner",

test/test_tags.py

@@ -0,0 +1,43 @@
+import sys
+import pytest
+import testlib_tags as testlib  # noqa
+
+
+def test_parse_tags():
+    from pytools.tag import parse_tag
+
+    def assert_same_as_python(tag_text):
+        assert parse_tag(tag_text) == eval(tag_text)
+
+    assert_same_as_python("testlib.FruitNameTag(testlib.MangoTag())")
+    assert_same_as_python("testlib.LocalAxisTag(0)")
+    assert_same_as_python('testlib.ColorTag("blue")')
+    assert_same_as_python('testlib.ColorTag(color="blue")')
+    assert_same_as_python("testlib.LocalAxisTag(axis=0)")
+
+    assert (parse_tag("testlib.FruitNameTag(.mango)",
+                     shortcuts={"mango": testlib.MangoTag(),
+                                "apple": testlib.AppleTag()})
+            == testlib.FruitNameTag(testlib.MangoTag()))
+
+    assert (parse_tag(".l.0",
+                     shortcuts={"l.0": testlib.LocalAxisTag(0)})
+            == testlib.LocalAxisTag(axis=0))
+
+
+def test_parse_tag_raises():
+    from pytools.tag import parse_tag
+
+    with pytest.raises(ValueError):
+        parse_tag('testlib.ColorTag(color="green",color="blue")')
+
+    with pytest.raises(TypeError):
+        parse_tag('testlib.ColorTag(typoed_kwarg_name="typo",color="blue")')
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        exec(sys.argv[1])
+    else:
+        from pytest import main
+        main([__file__])

test/testlib_tags.py

@@ -0,0 +1,26 @@
+from pytools.tag import Tag, UniqueTag
+
+
+class FruitNameTag(UniqueTag):
+    def __init__(self, tag):
+        self.tag = tag
+
+
+class MangoTag(Tag):
+    def __str__(self):
+        return "mango"
+
+
+class AppleTag(Tag):
+    def __str__(self):
+        return "apple"
+
+
+class ColorTag(Tag):
+    def __init__(self, color):
+        self.color = color
+
+
+class LocalAxisTag(Tag):
+    def __init__(self, axis):
+        self.axis = axis