diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8ea4dfe..b0a6f74a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: python-version: '3.x' - name: "Main Script" run: | - EXTRA_INSTALL="pymbolic" + EXTRA_INSTALL="pymbolic lark-parser" curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-pylint.sh . ./prepare-and-run-pylint.sh "$(basename $GITHUB_REPOSITORY)" test/test_*.py @@ -74,7 +74,7 @@ jobs: # AK, 2020-12-13 rm pytools/log.py - EXTRA_INSTALL="numpy" + EXTRA_INSTALL="numpy lark-parser" curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/build-and-test-py-project.sh . ./build-and-test-py-project.sh @@ -106,7 +106,7 @@ jobs: python-version: '3.x' - name: "Main Script" run: | - EXTRA_INSTALL="numpy" + EXTRA_INSTALL="numpy lark-parser" curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/ci-support.sh . ci-support.sh build_py_project_in_venv diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index be167b79..ac1670d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ Pytest: # AK, 2020-12-13 rm pytools/log.py - export EXTRA_INSTALL="numpy" + export EXTRA_INSTALL="numpy lark-parser" curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/build-and-test-py-project.sh . ./build-and-test-py-project.sh tags: @@ -50,7 +50,7 @@ Mypy: Pylint: script: - - EXTRA_INSTALL="pymbolic" + - EXTRA_INSTALL="pymbolic lark-parser" - py_version=3 - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-pylint.sh - . ./prepare-and-run-pylint.sh "$CI_PROJECT_NAME" test/test_*.py @@ -61,7 +61,7 @@ Pylint: Documentation: script: - - EXTRA_INSTALL="numpy" + - EXTRA_INSTALL="numpy lark-parser" - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/main/build-docs.sh - ". ./build-docs.sh" tags: diff --git a/MANIFEST.in b/MANIFEST.in index c6965e39..d539d515 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ include LICENSE include doc/*rst include doc/Makefile include doc/conf.py + +include pytools/*.lark diff --git a/pytools/tag.py b/pytools/tag.py index ca541ad6..7f45a127 100644 --- a/pytools/tag.py +++ b/pytools/tag.py @@ -1,5 +1,7 @@ from dataclasses import dataclass from typing import Tuple, Any, FrozenSet, Union, Iterable, TypeVar +import importlib + from pytools import memoize __copyright__ = """ @@ -40,6 +42,8 @@ .. autoclass:: Taggable .. autoclass:: Tag .. autoclass:: UniqueTag +.. autoclass:: ToPythonObjectMapper +.. autofunction:: parse_tag Supporting Functionality ------------------------ @@ -268,4 +272,127 @@ def without_tags(self: T_co, # }}} + +# {{{ parse + +class CallParams: + """ + Intermediate data structure for :class:`ToPythonObjectMapper`. + """ + def __init__(self, args, kwargs): + self.args = args + self.kwargs = kwargs + + +class ToPythonObjectMapper: + """ + Map a parsed tree to pythonic objects. + """ + def __init__(self, shortcuts, caller_globals): + super().__init__() + self.shortcuts = shortcuts + self.caller_globals = caller_globals + + def map_tag_from_python_class(self, cls, params): + try: + return cls(*params.args, **params.kwargs) + except TypeError as e: + raise TypeError(f"Error while instantiating '{cls.__name__}': {e}") + + def map_empty_args_params(self): + return CallParams((), {}) + + def map_args_only_params(self, args): + return CallParams(args, {}) + + def map_kwargs_only_params(self, kwargs): + return CallParams((), kwargs) + + def map_args_kwargs_params(self, args, kwargs): + return CallParams(args, kwargs) + + def map_name(self, tok): + return tok.value + + def map_top_level_module(self, modulename): + try: + return self.caller_globals[modulename] + except KeyError: + return importlib.import_module(modulename) + + def map_nested_module(self, module, child): + return getattr(module, child) + + def map_tag_class(self, module, classname): + return getattr(module, classname) + + def map_string(self, text): + return str(text) + + def map_int(self, text): + return int(text) + + def map_singleton_args(self, arg): + return (arg,) + + def map_args(self, args, arg): + return args + (arg,) + + def map_kwarg(self, name, arg): + return {name: arg} + + def map_kwargs(self, kwargs, kwarg): + assert len(kwarg) == 1 + (key, val), = kwarg.items() + if key in kwargs: + raise ValueError(f"keyword argument '{key}' repeated") + + updated_kwargs = kwargs.copy() + updated_kwargs[key] = val + return updated_kwargs + + def map_tag_from_shortcut(self, name): + name = name[1:] # remove the starting "." + return self.shortcuts[name] + + +@memoize +def construct_parser(): + from lark import Lark + return Lark.open("tags.lark", rel_to=__file__, parser="lalr", start="tag", + cache=True) + + +def parse_tag(tag_text, shortcuts={}): + """ + Parses a :class:`Tag` from a provided dotted name. + """ + import inspect + from lark import Transformer + + class ToPythonObjectMapperMixin(ToPythonObjectMapper, Transformer): + def _call_userfunc(self, tree, new_children=None): + """ + Flattens the arguments before feeding to the mapper methods. + """ + # Assumes tree is already transformed + children = new_children if new_children is not None else tree.children + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree.data, children, tree.meta) + else: + # flatten the args + return f(*children) + + parser = construct_parser() + + caller_globals = inspect.currentframe().f_back.f_globals + tag = ToPythonObjectMapperMixin(shortcuts, caller_globals).transform( + parser.parse(tag_text)) + + return tag + +# }}} + # vim: foldmethod=marker diff --git a/pytools/tags.lark b/pytools/tags.lark new file mode 100644 index 00000000..f6ee2480 --- /dev/null +++ b/pytools/tags.lark @@ -0,0 +1,35 @@ +tag: tag_class "(" params ")" -> map_tag_from_python_class + | SHORTCUT -> map_tag_from_shortcut + +params: -> map_empty_args_params + | args -> map_args_only_params + | kwargs -> map_kwargs_only_params + | args "," kwargs -> map_args_kwargs_params + +?kwargs: kwarg + | kwargs "," kwarg -> map_kwargs + +args: arg -> map_singleton_args + | args "," arg -> map_args + +kwarg: name "=" arg -> map_kwarg + +?arg: tag + | INT -> map_int + | ESCAPED_STRING -> map_string + +tag_class: module "." name -> map_tag_class + +module: name -> map_top_level_module + | module "." name -> map_nested_module + +name: CNAME -> map_name +SHORTCUT: "." ("_"|LETTER) ("_"|LETTER|DIGIT|".")* + +%import common.INT +%import common.ESCAPED_STRING +%import common.DIGIT +%import common.LETTER +%import common.CNAME +%import common.WS +%ignore WS diff --git a/setup.py b/setup.py index ed73fd64..af0b0cc5 100644 --- a/setup.py +++ b/setup.py @@ -41,9 +41,13 @@ "decorator>=3.2.0", "appdirs>=1.4.0", "numpy>=1.6.0", - "dataclasses>=0.7;python_version<='3.6'" + "dataclasses>=0.7;python_version<='3.6'", ], + extras_require={ + "tag_parsing": ["lark-parser"], + }, + package_data={"pytools": ["py.typed"]}, author="Andreas Kloeckner", diff --git a/test/test_tags.py b/test/test_tags.py new file mode 100644 index 00000000..dcd1b4d3 --- /dev/null +++ b/test/test_tags.py @@ -0,0 +1,43 @@ +import sys +import pytest +import testlib_tags as testlib # noqa + + +def test_parse_tags(): + from pytools.tag import parse_tag + + def assert_same_as_python(tag_text): + assert parse_tag(tag_text) == eval(tag_text) + + assert_same_as_python("testlib.FruitNameTag(testlib.MangoTag())") + assert_same_as_python("testlib.LocalAxisTag(0)") + assert_same_as_python('testlib.ColorTag("blue")') + assert_same_as_python('testlib.ColorTag(color="blue")') + assert_same_as_python("testlib.LocalAxisTag(axis=0)") + + assert (parse_tag("testlib.FruitNameTag(.mango)", + shortcuts={"mango": testlib.MangoTag(), + "apple": testlib.AppleTag()}) + == testlib.FruitNameTag(testlib.MangoTag())) + + assert (parse_tag(".l.0", + shortcuts={"l.0": testlib.LocalAxisTag(0)}) + == testlib.LocalAxisTag(axis=0)) + + +def test_parse_tag_raises(): + from pytools.tag import parse_tag + + with pytest.raises(ValueError): + parse_tag('testlib.ColorTag(color="green",color="blue")') + + with pytest.raises(TypeError): + parse_tag('testlib.ColorTag(typoed_kwarg_name="typo",color="blue")') + + +if __name__ == "__main__": + if len(sys.argv) > 1: + exec(sys.argv[1]) + else: + from pytest import main + main([__file__]) diff --git a/test/testlib_tags.py b/test/testlib_tags.py new file mode 100644 index 00000000..5bec114a --- /dev/null +++ b/test/testlib_tags.py @@ -0,0 +1,26 @@ +from pytools.tag import Tag, UniqueTag + + +class FruitNameTag(UniqueTag): + def __init__(self, tag): + self.tag = tag + + +class MangoTag(Tag): + def __str__(self): + return "mango" + + +class AppleTag(Tag): + def __str__(self): + return "apple" + + +class ColorTag(Tag): + def __init__(self, color): + self.color = color + + +class LocalAxisTag(Tag): + def __init__(self, axis): + self.axis = axis