diff --git a/ifex/model/ifex_ast_construction.py b/ifex/model/ifex_ast_construction.py index 79d9b08..4feb5c6 100644 --- a/ifex/model/ifex_ast_construction.py +++ b/ifex/model/ifex_ast_construction.py @@ -99,6 +99,7 @@ def ifex_ast_to_dict(node, debug_context="") -> OrderedDict: # Optional, otherwise the type-checking constructor would have caught the # error. + #print(f"node is: {node}\n") for f in fields(node): item = getattr(node, f.name) if not is_empty(item): diff --git a/other/franca/__init__.py b/other/franca/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/other/franca/franca_to_ifex.py b/other/franca/franca_to_ifex.py new file mode 100644 index 0000000..d944b97 --- /dev/null +++ b/other/franca/franca_to_ifex.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MBition GmbH. +# SPDX-License-Identifier: MPL-2.0 + +# This file is part of the IFEX project +# vim: tw=120 ts=4 et + +# Have to define a search path to submodule to make this work (might be rearranged later) +import os +import sys +mydir = os.path.dirname(__file__) +for p in ['pyfranca', 'pyfranca/pyfranca']: + if p not in sys.path: + sys.path.append(os.path.join(mydir,p)) + +import ifex.model.ifex_ast as ifex +import other.franca.pyfranca.pyfranca as pyfranca +import other.franca.rule_translator as m2m +import pyfranca.ast as franca +import re + +from ifex.model.ifex_ast_construction import add_constructors_to_ifex_ast_model, ifex_ast_as_yaml +from other.franca.rule_translator import Initialize, Constant, ListOf, Unsupported + +def array_type_name(francaitem): + return translate_type_name(francaitem) + '[]' # Unbounded arrays for now + +def translate_type_name(francaitem): + return translate_type(francaitem) + +def concat_comments(list): + return "\n".join(list) + +# If enumerator values are not given, we must use auto-generated values. +# IFEX model requires all enumerators to be given values. +enum_count = -1 +def reset_enumerator_counter(_ignored): + print("***Resetting enum counter") + global enum_count + enum_count = -1 + +def translate_enumerator_value(franca_int_value): + if franca_int_value is None: + global enum_count + enum_count = enum_count + 1 + return enum_count + return translate_integer_constant(franca_int_value) + +# Integer value is represented by an instance of IntegerValue class type, which has a "value" member +def translate_integer_constant(franca_int_value): + return franca_int_value.value + +# Tip: This translation table format is described in more detail in rule_translator.py +franca_to_ifex_mapping = { + + 'global_attribute_map': { + # Franca-name : IFEX-name + 'comments' : 'description', # FIXME allow transform also here, e.g. concat comments + 'extends' : None, # TODO + 'flags' : None + }, + + 'type_map': { + (franca.Interface, ifex.Interface) : [], + (franca.Package, ifex.Namespace) : [ + # TEMPORARY: Translates only the first interface + ('interfaces', 'interface', lambda x: x[0]), + ('typecollections', 'namespaces') ], + (franca.Method, ifex.Method) : [ + ('in_args', 'input'), + ('out_args', 'output'), + ('namespace', None) ], + (franca.Argument, ifex.Argument) : [ + ('type', 'datatype', translate_type_name), ], + (franca.Enumeration, ifex.Enumeration) : [ + (Initialize(reset_enumerator_counter), None), + ('enumerators', 'options'), + ('extends', Unsupported), + + # Franca only knows integer-based Enumerations so we hard-code the enumeration datatype to be + # int32 in the corresponding IFEX representation + (Constant('int32'), 'datatype') + ], + + (franca.Enumerator, ifex.Option) : [ + ('value', 'value', translate_enumerator_value) + ], + (franca.TypeCollection, ifex.Namespace) : [ + ('structs', 'structs'), + ('unions', None), # TODO - use the variant type on IFEX side, need to check its implementation first + ('arrays', 'typedefs'), + ('typedefs', 'typedefs') + ], + (franca.Struct, ifex.Struct) : [ + ('fields', 'members') + ], + (franca.StructField, ifex.Member) : [ + ('type', 'datatype', translate_type_name) + ] , + (franca.Array, ifex.Typedef) : [ + ('type', 'datatype', array_type_name) + ], + (franca.Attribute, ifex.Property) : [], + (franca.Import, ifex.Include) : [], + + # TODO: More mapping to do, much is not yet defined here + (franca.Package, ifex.Enumeration) : [], + (franca.Package, ifex.Struct) : [], + } + } + +# --- Map fundamental/built-in types --- + +type_translation = { + franca.Boolean : "boolean", + franca.ByteBuffer : "uint8[]", + franca.ComplexType : "opaque", # FIXME this is actually a struct reference? + franca.Double : "double", + franca.Float : "float", + franca.Int8 : "int8", + franca.Int16 : "int16", + franca.Int16 : "int16", + franca.Int32 : "int32", + franca.Int64 : "int64", + franca.String : "string", + franca.UInt8 : "uint8", + franca.UInt16 : "uint16", + franca.UInt32 : "uint32", + franca.UInt64 : "uint64", +} + +# ---------------------------------------------------------------------------- +# HELPER FUNCTIONS +# ---------------------------------------------------------------------------- + +def translate_type(t): + if type(t) is franca.Enumeration: + return t.name # FIXME use qualified name _, or change in the other place + if type(t) is franca.Reference: + return t.name + if type(t) is franca.Array: + # FIXME is size of array defined in FRANCA? + converted_type = translate_type(t.type) + converted_type = converted_type + '[]' + return converted_type + else: + t2 = type_translation.get(type(t)) + return t2 if t2 is not None else t + +# Rename fidl to ifex, for imports +def ifex_import_ref_from_fidl(fidl_file): + return re.sub('.fidl$', '.ifex', fidl_file) + +# ---------------------------------------------------------------------------- +# --- MAIN conversion function --- +# ---------------------------------------------------------------------------- + +# Build the Franca AST +def parse_franca(fidl_file): + processor = pyfranca.Processor() + return processor.import_file(fidl_file) # This returns the top level package + +# --- Script entry point --- + +if __name__ == '__main__': + + if len(sys.argv) != 2: + print(f"Usage: python {os.path.basename(__file__)} ") + sys.exit(1) + + # Add the type-checking constructor mixin + # FIXME Add this back later for strict checking + #add_constructors_to_ifex_ast_model() + + try: + # Parse franca input and create franca AST (top node is the Package definition) + franca_ast = parse_franca(sys.argv[1]) + + # Convert Franca AST to IFEX AST + ifex_ast = m2m.transform(franca_to_ifex_mapping, franca_ast) + + # Output as YAML + print(ifex_ast_as_yaml(ifex_ast)) + + except FileNotFoundError: + log("ERROR: File not found") + except Exception as e: + raise(e) + log("ERROR: An unexpected error occurred: {}".format(e)) + diff --git a/other/franca/pyfranca b/other/franca/pyfranca new file mode 160000 index 0000000..e847ef5 --- /dev/null +++ b/other/franca/pyfranca @@ -0,0 +1 @@ +Subproject commit e847ef5b419395dc0ea9f988e0fb49ffd16ff161 diff --git a/other/franca/rule_translator.py b/other/franca/rule_translator.py new file mode 100644 index 0000000..be5a47c --- /dev/null +++ b/other/franca/rule_translator.py @@ -0,0 +1,321 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MBition GmbH. +# SPDX-License-Identifier: MPL-2.0 + +# This file is part of the IFEX project +# vim: tw=120 ts=4 et + +""" +## rule_translator.py + +rule_translator implements a generic model-to-model translation function used to copy-and-transform values from one +hierarchical AST representation to another. It is driven by an input data structure that dewcribes the translation +rules to follow, and implemented by a generic function that can be used for many types of transformation. + +## Translation definition + +- The data structure (table) describes the mapping from the types (classes) of the input AST to the output AST +- Every type that is found in the input AST *should* have a mapping. There is not yet perfect error + reporting if something is missing, but it might be improved. +- For each class, the equivalent member variable that need to be mapped is listed. +- Member variable mappings are optional because any variable with Equal Name on each object + will be copied automatically (with possible transformation, *if* the input data is listed as a + complex type). +- Each attribute mapping can also optionally state the name of a transformation function (or lambda) + If no function is stated, the value will be mapped directly. Mapping means to follow the transformation + rules of the type-mapping table *if* the value is an instance of an AST class, and in other + cases simly copy the value as it is (typically a string, integer, etc.) +- In addition, it is possible to define global name-translations for attributes that are + equivalent but have different name in the two AST class families. +- To *ignore* an attribute, map it to the None value. + +See example table in rule_translator.py source for more information, or any of the implemented programs. + +""" + +from collections import OrderedDict +from dataclasses import dataclass +import os +import re +import sys + +# ----------------------------------------------------------------------------- +# Common Helpers +# ----------------------------------------------------------------------------- + +# These functions are likely to be reused by multiple m2m transformations and therefore provided here in a single +# location. + +def translate_integer_constant(franca_int_value): + return franca_int_value.value + +# ----------------------------------------------------------------------------- +# Translation Table Helper-objects +# ----------------------------------------------------------------------------- +# +# These classes help the table definition by defining something akin to a small DSL (Domain Specific Language) that aids +# us in expressing the translation table with things like "List Of" a certain type, or "Constant" when a given value is +# always supposed to be used, etc. Some ideas of representing the rules using python builtin primitives do not work. +# For example, using '[SomeClass]' to represent and array (list) of SomeClass is a valid statement in general, but +# does not work in our current translation table format because it is a key in a dict. Plain arrays are not a hashable +# value and therefore can't be used as a key. Similarly list(SomeClass) -> a type is not hashable. + +# (Use frozen dataclasses to make them hashable. The attributes are given values at construction time _only_.) +@dataclass(frozen=True) +class ListOf: + itemtype: type + +# Map to Unsupported to make a node type unsupported +@dataclass(frozen=True) +class Unsupported: + pass + +# To insert the same value for all translations +@dataclass(frozen=True) +class Constant: + value: int # Temporary, might be reassigned another type + +# To wrap a function that will be called at this stage in the attribute mapping +@dataclass(frozen=True) +class Initialize: + func: callable + pass + + +# ----------------------------------------------------------------------------- +# Translation Table - Example, not used. The table be provided instead by the program +# that uses this module) +# ----------------------------------------------------------------------------- + +example = """ +example_mapping = { + + # global_attribute_map: Attributes with identical name are normally automatically mapped. If attributes have + # different names we can avoid repeated mapping definitions by still defining them as equivalent in the + # global_attribute_map. Attributes defined here ill be mapped in *all* classes. Note: To ignore an attribute, + # map it to None! + + 'global_attribute_map': { + # (Attribute in FROM-class on the left, attribute in TO-class on the right) + 'this' : 'that', + 'something_not_needed' : None + }, + + # type_map: Here follows Type (class) mappings with optional local attribute mappings + 'type_map': { + (inmodule.ASTClass1, outmodule.MyASTClass) : + # followed by array of attribute mapping + + # Here an array of tuples is used. (Format is subject to change) + # (FROM-class on the left, TO-class on the right) + # *optional* transformation function as third argument + [ ('thiss', 'thatt'), + ('name', 'thename', capitalize_name_string), + ('zero_based_counter', 'one_based_counter', lambda x : x + 1), + ('thing', None) + ] + + # Equally named types have no magic - it is still required to + # define that they shall be transformed/copied over. + (inmodule.AnotherType, outmodule.Anothertype), [ + # Use a Constant object to always set the same value in target attribute + (Constant('int32'), 'datatype') + ], + # ListOf and other features are not yet implemented -> when the need arises + } +} +""" + +# ----------------------------------------------------------------------------- + +# In the following table it is possible to list additional functions that are required but cannot be covered by the +# one-to-one object mapping above. A typical example is to recursively loop over a major container, *and* its children +# containers create a flat list of items. Non-obvious mappings can be handled by processing the AST several times. +# Example: if in the input AST has typedefs defined on the global scope, as well as inside of a namespace/interface, but +# in the output AST we want them all collected on a global scope, then the direct mapping between AST objects does not +# apply well since that only creates a result that is analogous to the structure of the input AST. + +# NOTE: This is not yet implemented -> when the need arises +ast_translation = { + +} + +# Other commonalities +# +# Most X-to-IFEX transformations will use a table mapping the fundamental/built-in types +# to IFEX types. +# Example, which shows that as of now IFEX types are not strongly typed, but represented by string) +# E.g. type_translation = { franca.Boolean : "boolean", franca.ByteBuffer : "uint8[]", ... } etc. + +# ---------------------------------------------------------------------------- +# HELPER FUNCTIONS +# ---------------------------------------------------------------------------- + +# TODO - add better logging +def _log(level, string): + pass + #print(f"{level}: {string}") + +def is_builtin(x): + return x.__class__.__module__ == 'builtins' + +# This is really supposed to check if the instance is one of the AST classes, or possibly it could check if it is a class +# defined in the mapping table. For now, however, this simple check for "not builtin" works well enough. +def is_composite_object(mapping_table, x): + return not is_builtin(x) + +# FIXME: Unused, but could be used for error checking +def has_mapping(mapping_table, x): + return mapping_table['type_map'].get(x.__class__) is not None + +# flatmap: Call function for each item in input_array, and flatten the result +# into one array. The passed function is expected to return an array for each call. +def flatmap(function, input_array): + return [y for x in input_array for y in function(x)] + +def underscore_combine_name(parent, item): + return parent + "_" + item + +# This function is used by the general translation to handle multiple mappings with the same target attribute. +# We don't want to overwrite and destroy the previous value with a new one, and if the target is a list +# then it is useful to be able to add to that list at multiple occasions -> append to it. +def set_attr(attrs_dict, attr_key, attr_value): + if attr_key in attrs_dict: + value = attrs_dict[attr_key] + + # If it's a list, we can add to it instead of overwriting: + if isinstance(value, list): + value.append(attr_value) + attrs_dict[attr_key] = value + return + else: + _log("ERR", """Attribute {attr_key} already has a scalar (non-list) value. Check for multiple translations + mapping to this one. We should not overwrite it, and since it is not a list type, we can't append.""") + _log("DEBUG" "Target value {value} was ignored.") + return + + attrs_dict[attr_key] = attr_value + +# ---------------------------------------------------------------------------- +# --- MAIN conversion function --- +# ---------------------------------------------------------------------------- + +def transform(mapping_table, input_obj): + + # Builtin types (str, int, ...) are assumed to be just values to copy without any change + if is_builtin(input_obj): + return input_obj + + # Find a translation rule in the metadata + # Uses linear-search in mapping table until we find something matching input object. + # Since the translation table is reasonably short, it should be OK for now. + for (from_class, to_class), mappings in mapping_table['type_map'].items(): + + # Does this transformation rule match input object? + if from_class == input_obj.__class__: + _log("INFO", f"Type mapping found: {from_class=} -> {to_class=}") + + # Comment: Here we might create an empty instance of the class and fill it with values using setattr(), but + # that won't work since the redesign using dataclasses. The AST classes now have a default constructor that + # requires all mandatory fields to be specified when an instance is created. Therefore we are forced to + # follow this approach: Gather all attributes in a dict and pass it into the constructor at the end using + # python's dict to keyword-arguments capability. + + attrs = {} + + # To remember the args we have converted + done_attrs = set() + + # First loop: Perform the explicitly defined attribute conversions + # for those that are specified in the translation table. + + for input_attr, output_attr, *transform_attribute in mappings: + + transform_attribute = transform_attribute[0] if transform_attribute else lambda x: x + + # A init/prep function was defined. The "output_attr" is here misleadingly named. + # It is (optionally) used to define a parameter to be passed to the function + if isinstance(input_attr, Initialize): + input_attr.func(output_attr) + continue + + _log("INFO", f"Attribute mapping found: {input_attr=} -> {output_attr=} with {transform_attribute=}") + if output_attr is None: + _log("DEBUG", f"Ignoring {input_attr=} for {type(input_obj)} because it was mapped to None") + continue + + if output_attr is Unsupported: + if value is not None: + _log("ERR", f"Attribute {input_attr} has a value in {type(input_obj)}:{input_obj.name} but the feature ({input_attr}) is unsupported. ({value=})") + continue + + # Get input value, or assign constant if so specified + if isinstance(input_attr, Constant): + value = input_attr.value + else: + value = getattr(input_obj, input_attr) + + # OrderedDict is used at least by Franca AST + if isinstance(value, OrderedDict): + newval = [transform(mapping_table, item) for name, item in value.items()] + set_attr(attrs, output_attr, newval) + + elif isinstance(value, list): + newval = [transform(mapping_table, item) for item in value] + set_attr(attrs, output_attr, newval) + + else: # Plain attribute -> use transform_attribute if it was defined + set_attr(attrs, output_attr, transform_attribute(value)) + + # Mark this attribute as done + done_attrs.add(input_attr) + + # Second loop: Any attributes that have the _same name_ in the input and output classes are assumed to be + # the mappable to eachother. They do not need to be listed in the translation table unless they need a + # custom transformation. Here we find all matching names here and map them (with recursive transformation, + # as needed), but of course skip all attributes that have been handled already (done_attrs) + + global_attribute_map = mapping_table['global_attribute_map'] + for attr, value in vars(input_obj).items(): + + # Skip already done items + if attr in done_attrs: + continue + + # Translate attribute name according to global rules, if that is defined. + if attr in global_attribute_map: + attr = global_attribute_map.get(attr) + + # Check if to_class has this attribute with the same name + # => Things get a bit ugly here because of the use of dataclasses as explained above + if to_class.__dataclass_fields__.__contains__(attr): + _log("DEBUG", f"Global or same-name auto-conversion for {attr=} from {from_class.__name__} to {to_class.__name__}\n") + + if isinstance(value, OrderedDict): + # A dict of name/value pairs -> map it to a list in the output but iterate over the actual items, not the keys + value = [transform(mapping_table, item[1]) for item in value.items()] + + elif isinstance(value, list): + # List value -> translate each contained object, and make a list of the results. + value = [transform(mapping_table, item) for item in value] + + elif is_composite_object(mapping_table, value): + # A single value of AST node type -> recurse to translate this object + value = transform(mapping_table, value) + + # else: Value is a single and simple type (string/int/ etc.) -> just copy it as it is + + # Set the value we determined for this attribute for the (soon to be created) to_class object + set_attr(attrs, attr, value) + + # No match found in to_class + elif attr is not None: + _log("WARN", f"Attribute '{attr}' from Franca:{input_obj.__class__.__name__} was not used in IFEX:{to_class.__name__}") + + # Instantiate to_class object, and return + _log("DEBUG", f"Creating and returning object of type {to_class} with {attrs=}") + return to_class(**attrs) + + no_rule = f"No translation rule found for object {input_obj} of class {input_obj.__class__.__name__}" + _log("ERR:", "no_rule") + raise TypeError(no_rule) + diff --git a/requirements.txt b/requirements.txt index 2d963f9..a82b46a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ dacite==1.6.0 jsonschema>=4.20 lark==1.1.9 setuptools +ply>=3.11