From d149b4cb375769b6f2e9db50ab27c633a73e95aa Mon Sep 17 00:00:00 2001 From: Dariusz Suchojad Date: Sun, 17 Sep 2023 10:13:36 +0200 Subject: [PATCH 1/4] GH #755 - Adding adapter services. --- code/zato-common/src/zato/common/marshal_/api.py | 5 +++-- code/zato-common/src/zato/common/marshal_/model.py | 12 ++++++++++++ code/zato-common/src/zato/common/typing_.py | 5 +++++ .../src/zato/server/connection/http_soap/outgoing.py | 11 +++-------- code/zato-server/src/zato/server/service/__init__.py | 9 ++++++++- 5 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 code/zato-common/src/zato/common/marshal_/model.py diff --git a/code/zato-common/src/zato/common/marshal_/api.py b/code/zato-common/src/zato/common/marshal_/api.py index 69926f52e1..53e3932c61 100644 --- a/code/zato-common/src/zato/common/marshal_/api.py +++ b/code/zato-common/src/zato/common/marshal_/api.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -Copyright (C) 2022, Zato Source s.r.o. https://zato.io +Copyright (C) 2023, Zato Source s.r.o. https://zato.io Licensed under LGPLv3, see LICENSE.txt for terms and conditions. """ @@ -30,6 +30,7 @@ class _Sentinel: # Zato from zato.common.api import ZatoNotGiven +from zato.common.marshal_.model import BaseModel from zato.common.typing_ import cast_, extract_from_union, is_union # ################################################################################################################################ @@ -90,7 +91,7 @@ def extract_model_class(field_type:'Field') -> 'Model | None': # ################################################################################################################################ # ################################################################################################################################ -class Model: +class Model(BaseModel): __name__: 'str' after_created = None diff --git a/code/zato-common/src/zato/common/marshal_/model.py b/code/zato-common/src/zato/common/marshal_/model.py new file mode 100644 index 0000000000..e5de2af70d --- /dev/null +++ b/code/zato-common/src/zato/common/marshal_/model.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +""" +Copyright (C) 2023, Zato Source s.r.o. https://zato.io + +Licensed under LGPLv3, see LICENSE.txt for terms and conditions. +""" + +class BaseModel: + """ This is a base class for actual models. On its own, it does not implement anything. + It is used only for type hints. + """ diff --git a/code/zato-common/src/zato/common/typing_.py b/code/zato-common/src/zato/common/typing_.py index 599904a912..9afff5cd01 100644 --- a/code/zato-common/src/zato/common/typing_.py +++ b/code/zato-common/src/zato/common/typing_.py @@ -42,6 +42,9 @@ # stdlib from dataclasses import * # type: ignore +# Zato +from zato.common.marshal_.model import BaseModel + # ################################################################################################################################ # ################################################################################################################################ @@ -106,6 +109,8 @@ iterator_ = iterator_ iobytes_ = io_[bytes] listnone = anylistnone +model = type_[BaseModel] +modelnone = optional[type_[BaseModel]] noreturn = noreturn set_ = set_ stranydict = dict_[str, any_] diff --git a/code/zato-server/src/zato/server/connection/http_soap/outgoing.py b/code/zato-server/src/zato/server/connection/http_soap/outgoing.py index 7595dfd7af..d01d2aaafd 100644 --- a/code/zato-server/src/zato/server/connection/http_soap/outgoing.py +++ b/code/zato-server/src/zato/server/connection/http_soap/outgoing.py @@ -563,21 +563,16 @@ def upload( # ################################################################################################################################ - def get_by_query_id( + def get_by_query( self, *, cid, # type: str - id, # type: any_ model=None, # type: type_[Model] | None callback, # type: callnone + params=None # type: strdictnone ) -> 'any_': - # .. build the query parameters .. - params:'stranydict' = { - 'id': id, - } - - # .. invoke the system .. + # Invoke the system .. try: response:'Response' = self.get(cid, params) except Exception as e: diff --git a/code/zato-server/src/zato/server/service/__init__.py b/code/zato-server/src/zato/server/service/__init__.py index 4bf86f1d84..eaa435d19c 100644 --- a/code/zato-server/src/zato/server/service/__init__.py +++ b/code/zato-server/src/zato/server/service/__init__.py @@ -489,10 +489,16 @@ def __init__( self.environ = Bunch() self.request = Request(self) # type: Request self.response = Response(self.logger) # type: ignore - self.user_config = Bunch() self.has_validate_input = False self.has_validate_output = False + # This is where user configuration is kept + self.config = Bunch() + + # This is kept for backward compatibility with code that uses self.user_config in services. + # Only self.config should be used in new services. + self.user_config = Bunch() + self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late @@ -1322,6 +1328,7 @@ def update( service.wsgi_environ = wsgi_environ or {} service.job_type = job_type service.translate = server.kvdb.translate # type: ignore + service.config = server.user_config service.user_config = server.user_config service.static_config = server.static_config service.time = server.time_util From 937b6248027a78f933b42098248ab189145d2734 Mon Sep 17 00:00:00 2001 From: Dariusz Suchojad Date: Sun, 17 Sep 2023 13:44:55 +0200 Subject: [PATCH 2/4] GH #1125 - Adding Model.build_model_from_flat_input. --- .../src/zato/common/marshal_/api.py | 15 ++++++- code/zato-common/src/zato/common/typing_.py | 1 + .../server/connection/http_soap/outgoing.py | 19 ++++---- .../src/zato/server/service/store.py | 45 ++++++++++++++++++- 4 files changed, 69 insertions(+), 11 deletions(-) diff --git a/code/zato-common/src/zato/common/marshal_/api.py b/code/zato-common/src/zato/common/marshal_/api.py index 53e3932c61..dedf4ca38e 100644 --- a/code/zato-common/src/zato/common/marshal_/api.py +++ b/code/zato-common/src/zato/common/marshal_/api.py @@ -38,7 +38,7 @@ class _Sentinel: if 0: from dataclasses import Field - from zato.common.typing_ import any_, anydict, boolnone, dictnone, intnone, optional + from zato.common.typing_ import any_, anydict, boolnone, dictnone, intnone, optional, tuplist, type_ from zato.server.service import Service Field = Field @@ -133,6 +133,14 @@ def clone(self) -> 'any_': out = self.__class__._zato_from_dict(None, data) return out + @staticmethod + def build_model_from_flat_input(name:'str', input:'str | tuplist') -> 'type_[BaseModel]': + + class _Model(BaseModel): + pass + + return _Model + # ################################################################################################################################ # ################################################################################################################################ @@ -223,6 +231,11 @@ def init(self): self.has_init = dataclass_params.init if dataclass_params else False self.attrs_container = self.init_attrs if self.has_init else self.setattr_attrs + + # This may be a string-only definition .. + #if isinstance(self.DataClass, str): + # self.fields = + self.fields = getattr(self.DataClass, _FIELDS) # type: dictnone # ################################################################################################################################ diff --git a/code/zato-common/src/zato/common/typing_.py b/code/zato-common/src/zato/common/typing_.py index 9afff5cd01..aed1507ef2 100644 --- a/code/zato-common/src/zato/common/typing_.py +++ b/code/zato-common/src/zato/common/typing_.py @@ -141,6 +141,7 @@ textio_ = textio_ textionone = textio_ tuple_ = tuple_ +tuplist = union_[anylist, anytuple] tupnone = optional[anytuple] type_ = type_ typealias_ = typealias_ diff --git a/code/zato-server/src/zato/server/connection/http_soap/outgoing.py b/code/zato-server/src/zato/server/connection/http_soap/outgoing.py index d01d2aaafd..d2386f7ea4 100644 --- a/code/zato-server/src/zato/server/connection/http_soap/outgoing.py +++ b/code/zato-server/src/zato/server/connection/http_soap/outgoing.py @@ -563,24 +563,27 @@ def upload( # ################################################################################################################################ - def get_by_query( + def rest_call( self, *, - cid, # type: str - model=None, # type: type_[Model] | None - callback, # type: callnone - params=None # type: strdictnone + cid, # type: str + model=None, # type: type_[Model] | None + callback, # type: callnone + params=None, # type: strdictnone + method='', # type: str + log_response=True, # type: bool ) -> 'any_': # Invoke the system .. try: - response:'Response' = self.get(cid, params) + response:'Response' = self.http_request(method, cid, params=params) except Exception as e: logger.warn('Caught an exception -> %s', e) else: - # .. log what we received .. - logger.info('Get By Query ID response received -> %s', response.text) + # .. optionally, log what we received .. + if log_response: + logger.info('REST call response received -> %s', response.text) if not response.ok: raise Exception(response.text) diff --git a/code/zato-server/src/zato/server/service/store.py b/code/zato-server/src/zato/server/service/store.py index 006f740064..83d90a4775 100644 --- a/code/zato-server/src/zato/server/service/store.py +++ b/code/zato-server/src/zato/server/service/store.py @@ -484,12 +484,53 @@ class _Class_SimpleIO: has_input_data_class = self._has_io_data_class(class_, sio_input, 'Input') has_output_data_class = self._has_io_data_class(class_, sio_output, 'Output') + # If either input or output is a dataclass but the other one is not, + # we need to turn the latter into a dataclass as well. + + # We are here if output is a dataclass .. + if has_output_data_class: + + # .. but input is not and it should be .. + if (not has_input_data_class) and sio_input: + + # .. create a name for the dynamically-generated input model class .. + name = str(class_) + name = name.replace('.', '_') + name += '_AutoInput' + + # .. generate the input model class now .. + sio_input = DataClassModel.build_model_from_flat_input(name, sio_input) + + # .. and assign it as input. + if 'ZZZ' in name: + name + name + + # We are here if input is a dataclass .. + if has_input_data_class: + + # .. but output is not and it should be. + if (not has_output_data_class) and sio_output: + + # .. create a name for the dynamically-generated output model class .. + name = str(class_) + name = name.replace('.', '_') + name += '_AutoOutput' + + # .. generate the input model class now .. + sio_output = DataClassModel.build_model_from_flat_input(name, sio_output) + + # .. and assign it as output. + if 'ZZZ' in name: + name + name + if has_input_data_class or has_output_data_class: SIOClass = DataClassSimpleIO else: - SIOClass = CySimpleIO + SIOClass = CySimpleIO # type: ignore - _ = SIOClass.attach_sio(service_store.server, service_store.server.sio_config, class_) + _ = SIOClass.attach_sio(service_store.server, service_store.server.sio_config, class_) # type: ignore # May be None during unit-tests - not every test provides it. if service_store: From fe9d6c559110d73365c797d4dc72147619c40b2e Mon Sep 17 00:00:00 2001 From: Dariusz Suchojad Date: Sun, 17 Sep 2023 17:32:36 +0200 Subject: [PATCH 3/4] GH #1125 - Working on auto-SIO. --- .../src/zato/common/marshal_/api.py | 60 +++++++++++++++---- .../zato/common/marshall_/test_build_model.py | 56 +++++++++++++++++ code/zato-cy/src/zato/cy/simpleio.py | 4 +- .../src/zato/server/service/store.py | 36 +++++++---- 4 files changed, 132 insertions(+), 24 deletions(-) create mode 100644 code/zato-common/test/zato/common/marshall_/test_build_model.py diff --git a/code/zato-common/src/zato/common/marshal_/api.py b/code/zato-common/src/zato/common/marshal_/api.py index dedf4ca38e..ec320fa900 100644 --- a/code/zato-common/src/zato/common/marshal_/api.py +++ b/code/zato-common/src/zato/common/marshal_/api.py @@ -7,7 +7,7 @@ """ # stdlib -from dataclasses import asdict, _FIELDS, MISSING, _PARAMS # type: ignore +from dataclasses import asdict, _FIELDS, make_dataclass, MISSING, _PARAMS # type: ignore from http.client import BAD_REQUEST from inspect import isclass from typing import Any @@ -37,8 +37,11 @@ class _Sentinel: # ################################################################################################################################ if 0: + from bunch import Bunch from dataclasses import Field from zato.common.typing_ import any_, anydict, boolnone, dictnone, intnone, optional, tuplist, type_ + from zato.simpleio import SIOServerConfig + from zato.server.base.parallel import ParallelServer from zato.server.service import Service Field = Field @@ -134,12 +137,54 @@ def clone(self) -> 'any_': return out @staticmethod - def build_model_from_flat_input(name:'str', input:'str | tuplist') -> 'type_[BaseModel]': + def build_model_from_flat_input( + server, # type: ParallelServer + sio_server_config, # type: ignore + _CySimpleIO, # type: ignore + name, # type: str + input, # type: str | tuplist + ) -> 'type_[BaseModel]': - class _Model(BaseModel): - pass + # Local imports + from zato.simpleio import is_sio_bool, is_sio_int - return _Model + # Local aliases + model_fields = [] + + # Make sure this is a list-like container .. + if isinstance(input, str): + input = [input] + + # .. build an actual SIO handler .. + _cy_simple_io = _CySimpleIO(server, sio_server_config, input) # type: ignore + + # .. now, go through everything we have on input .. + for item in input: + + # .. find out if this is a required element or not .. + is_optional = item.startswith('-') + is_required = not is_optional + + # .. turn each element input into a Cython-based one .. + sio_elem = _cy_simple_io.convert_to_elem_instance(item, is_required) # type: ignore + + # .. check if it is not a string .. + is_int:'bool' = is_sio_int(sio_elem) + is_bool:'bool' = is_sio_bool(sio_elem) + + # .. turn the type into a model-compatible name .. + if is_int: + _model_type = int + elif is_bool: + _model_type = bool + else: + _model_type = str + + # .. append a model-compatible definition of this field for later use .. + model_fields.append((sio_elem.name, _model_type)) + + model = make_dataclass(name, model_fields, bases=(Model,)) + return model # type: ignore # ################################################################################################################################ # ################################################################################################################################ @@ -231,11 +276,6 @@ def init(self): self.has_init = dataclass_params.init if dataclass_params else False self.attrs_container = self.init_attrs if self.has_init else self.setattr_attrs - - # This may be a string-only definition .. - #if isinstance(self.DataClass, str): - # self.fields = - self.fields = getattr(self.DataClass, _FIELDS) # type: dictnone # ################################################################################################################################ diff --git a/code/zato-common/test/zato/common/marshall_/test_build_model.py b/code/zato-common/test/zato/common/marshall_/test_build_model.py new file mode 100644 index 0000000000..d65262a5a6 --- /dev/null +++ b/code/zato-common/test/zato/common/marshall_/test_build_model.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +""" +Copyright (C) 2023, Zato Source s.r.o. https://zato.io + +Licensed under LGPLv3, see LICENSE.txt for terms and conditions. +""" + +# Zato +from zato.common.marshal_.api import Model +from zato.common.test import BaseSIOTestCase +from zato.common.typing_ import cast_ + +# ################################################################################################################################ +# ################################################################################################################################ + +if 0: + from zato.server.base.parallel import ParallelServer + +# ################################################################################################################################ +# ################################################################################################################################ + +class BuildModelTestCase(BaseSIOTestCase): + + def xtest_build_from_string(self): + + name = 'Test_Build_From_String' + input = 'abc' + + server = cast_('ParallelServer', None) + + model = Model.build_model_from_flat_input(server, name, input) + model + +# ################################################################################################################################ + + def xtest_build_from_list(self): + + name = 'Test_Build_From_List' + input = ['id', 'name', 'user_id, is_active'] + +# ################################################################################################################################ + + def xtest_build_from_tuple(self): + + name = 'Test_Build_From_List' + input = ('id', 'name', 'user_id, is_active') + +# ################################################################################################################################ +# ################################################################################################################################ + +if __name__ == '__main__': + _ = main() + +# ################################################################################################################################ +# ################################################################################################################################ diff --git a/code/zato-cy/src/zato/cy/simpleio.py b/code/zato-cy/src/zato/cy/simpleio.py index 9cf6ff688a..9d58e85a94 100644 --- a/code/zato-cy/src/zato/cy/simpleio.py +++ b/code/zato-cy/src/zato/cy/simpleio.py @@ -1570,7 +1570,7 @@ def build(self, class_:object): # ################################################################################################################################ @cy.returns(Elem) - def _convert_to_elem_instance(self, elem_name, is_required:cy.bint) -> Elem: + def convert_to_elem_instance(self, elem_name, is_required:cy.bint) -> Elem: # The element we return, at this point we do not know what its exact subtype will be _elem:Elem @@ -1717,7 +1717,7 @@ def _build_io_elems(self, container, class_): if isinstance(initial_elem, Elem): elem = initial_elem else: - elem = self._convert_to_elem_instance(initial_elem, is_required) + elem = self.convert_to_elem_instance(initial_elem, is_required) # By default all Elem instances are required so we need # to potentially overwrite it with the actual is_required value. diff --git a/code/zato-server/src/zato/server/service/store.py b/code/zato-server/src/zato/server/service/store.py index 83d90a4775..b0c337b765 100644 --- a/code/zato-server/src/zato/server/service/store.py +++ b/code/zato-server/src/zato/server/service/store.py @@ -431,6 +431,9 @@ def _has_io_data_class( def set_up_class_attributes(self, class_:'type[Service]', service_store:'ServiceStore') -> 'None': + # Local aliases + _Class_SimpleIO = None # type: ignore + # Set up enforcement of what other services a given service can invoke try: class_.invokes @@ -494,17 +497,22 @@ class _Class_SimpleIO: if (not has_input_data_class) and sio_input: # .. create a name for the dynamically-generated input model class .. - name = str(class_) + name = class_.__module__ + '_' + class_.__name__ name = name.replace('.', '_') name += '_AutoInput' # .. generate the input model class now .. - sio_input = DataClassModel.build_model_from_flat_input(name, sio_input) + model_input = DataClassModel.build_model_from_flat_input( + service_store.server, + service_store.server.sio_config, + CySimpleIO, + name, + sio_input + ) # .. and assign it as input. - if 'ZZZ' in name: - name - name + if _Class_SimpleIO: + _Class_SimpleIO.input = model_input # type: ignore # We are here if input is a dataclass .. if has_input_data_class: @@ -513,17 +521,21 @@ class _Class_SimpleIO: if (not has_output_data_class) and sio_output: # .. create a name for the dynamically-generated output model class .. - name = str(class_) + name = class_.__module__ + '_' + class_.__name__ name = name.replace('.', '_') name += '_AutoOutput' # .. generate the input model class now .. - sio_output = DataClassModel.build_model_from_flat_input(name, sio_output) - - # .. and assign it as output. - if 'ZZZ' in name: - name - name + model_output = DataClassModel.build_model_from_flat_input( + service_store.server, + service_store.server.sio_config, + CySimpleIO, + name, + sio_output + ) + + if _Class_SimpleIO: + _Class_SimpleIO.output = model_output # type: ignore if has_input_data_class or has_output_data_class: SIOClass = DataClassSimpleIO From fdd051c95a213aa000382204ad75cdbe2a36f7ba Mon Sep 17 00:00:00 2001 From: Dariusz Suchojad Date: Sun, 17 Sep 2023 18:12:48 +0200 Subject: [PATCH 4/4] GH #1125 - Adding type hints. --- .../src/zato/common/marshal_/api.py | 2 - .../zato/common/marshall_/test_build_model.py | 56 ------------------- 2 files changed, 58 deletions(-) delete mode 100644 code/zato-common/test/zato/common/marshall_/test_build_model.py diff --git a/code/zato-common/src/zato/common/marshal_/api.py b/code/zato-common/src/zato/common/marshal_/api.py index ec320fa900..bd83cc75e3 100644 --- a/code/zato-common/src/zato/common/marshal_/api.py +++ b/code/zato-common/src/zato/common/marshal_/api.py @@ -37,10 +37,8 @@ class _Sentinel: # ################################################################################################################################ if 0: - from bunch import Bunch from dataclasses import Field from zato.common.typing_ import any_, anydict, boolnone, dictnone, intnone, optional, tuplist, type_ - from zato.simpleio import SIOServerConfig from zato.server.base.parallel import ParallelServer from zato.server.service import Service diff --git a/code/zato-common/test/zato/common/marshall_/test_build_model.py b/code/zato-common/test/zato/common/marshall_/test_build_model.py deleted file mode 100644 index d65262a5a6..0000000000 --- a/code/zato-common/test/zato/common/marshall_/test_build_model.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright (C) 2023, Zato Source s.r.o. https://zato.io - -Licensed under LGPLv3, see LICENSE.txt for terms and conditions. -""" - -# Zato -from zato.common.marshal_.api import Model -from zato.common.test import BaseSIOTestCase -from zato.common.typing_ import cast_ - -# ################################################################################################################################ -# ################################################################################################################################ - -if 0: - from zato.server.base.parallel import ParallelServer - -# ################################################################################################################################ -# ################################################################################################################################ - -class BuildModelTestCase(BaseSIOTestCase): - - def xtest_build_from_string(self): - - name = 'Test_Build_From_String' - input = 'abc' - - server = cast_('ParallelServer', None) - - model = Model.build_model_from_flat_input(server, name, input) - model - -# ################################################################################################################################ - - def xtest_build_from_list(self): - - name = 'Test_Build_From_List' - input = ['id', 'name', 'user_id, is_active'] - -# ################################################################################################################################ - - def xtest_build_from_tuple(self): - - name = 'Test_Build_From_List' - input = ('id', 'name', 'user_id, is_active') - -# ################################################################################################################################ -# ################################################################################################################################ - -if __name__ == '__main__': - _ = main() - -# ################################################################################################################################ -# ################################################################################################################################