From 1001177a858d7d7fb8043cbc25c4c5eac893dfa9 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 6 Jun 2023 16:52:37 +0300 Subject: [PATCH 01/26] used new models --- backend/endpoints/api/v1/restapi.py | 213 +++++++++++++++++----------- 1 file changed, 132 insertions(+), 81 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 3fe8c195..44c78287 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -36,7 +36,8 @@ from dal.models.role import Role from dal.movaidb import MovaiDB from dal.scopes.application import Application -from dal.scopes.callback import Callback +#from dal.scopes.callback import Callback +from dal.new_models import Callback, cache from dal.scopes.configuration import Configuration from dal.scopes.flow import Flow from dal.scopes.form import Form @@ -132,7 +133,8 @@ async def cloud_func(self, request): callback = GD_Callback(callback_name, self.node_name, "cloud", False) # Check User permissions - scope_obj = self.scope_classes["Callback"](name=callback_name) + #scope_obj = self.scope_classes["Callback"](name=callback_name) + scope_obj = Callback(callback_name) if not scope_obj.has_permission(request.get("user"), "execute", app_name): raise ValueError("User does not have permission") @@ -721,17 +723,41 @@ async def get_scope(self, request: web.Request) -> web.Response: scope = request.match_info.get("scope") _id = request.match_info.get("name", False) + LOGGER.warning("Calling get_scope with scope: %s and id: %s", scope, _id) if _id: try: scope_obj = self.scope_classes[scope](_id) - except Exception: + except Exception as e: + LOGGER.error(f"caught error while creating scope object, exception: {e}") raise web.HTTPNotFound(reason="Required scope not found.") # Check User permissions if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - scope_result = MovaiDB().get({scope: {_id: "**"}}) + if scope.lower() in ["callback", "node"]: + if scope.lower() == "callback": + key = f"Movai:Callback:{_id}:__UNVERSIONED__" + if key in cache: + LOGGER.error(f" ############## retreiving from cache {key}") + scope_obj = cache[key] + else: + scope_obj = Callback(_id) + LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") + cache[key] = scope_obj + elif scope.lower() == "node": + from dal.new_models import Node + key = f"Movai:Node:{_id}:__UNVERSIONED__" + if key in cache: + scope_obj = cache[key] + else: + scope_obj = Node(_id) + LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") + cache[key] = scope_obj + + scope_result = scope_obj.dict() + else: + scope_result = MovaiDB().get({scope: {_id: "**"}}) result = scope_result[scope][_id] # If Scope User add permissions list @@ -745,7 +771,18 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - scope_result = MovaiDB().get_by_args(scope) + if scope.lower() in ["callback", "node"]: + if scope == "Callback": + objs = Callback.select() + else: + from dal.new_models import Node + objs = Node.select() + LOGGER.error(f"importing from new models {objs}") + scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} + LOGGER.error(scope_result) + scope_result = {scope: scope_result} + else: + scope_result = MovaiDB().get_by_args(scope) result = scope_result.get(scope, {}) if not result: @@ -887,91 +924,105 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: - label = data["data"].get("Label") - scope_class = self.scope_classes.get(scope) - struct = scope_class(label, new=True) - struct.Label = ( - label # just for now, this wont be needed when we merge branch "labeling" - ) - _id = struct.name - obj_created = _id - - scope_obj = scope_class(name=_id) + if scope.lower() == "callback": + scope_obj = Callback(**{"Callback": {label: data["data"]}}) + else: + label = data["data"].get("Label") + scope_class = self.scope_classes.get(scope) + struct = scope_class(label, new=True) + struct.Label = ( + label # just for now, this wont be needed when we merge branch "labeling" + ) + _id = struct.name + obj_created = _id + scope_obj = scope_class(name=_id) except Exception: raise web.HTTPBadRequest(reason="This already exists") else: - # Check if scope exists - try: - scope_class = self.scope_classes.get(scope) - scope_obj = scope_class(name=_id) - except Exception: - raise web.HTTPNotFound(reason="Scope object not found") + if scope.lower() == "callback": + # check if exist + Callback(_id) + label = data["data"].get("Label") + scope_obj = Callback(**{"Callback": {label: data["data"]}}) + else: + # Check if scope exists + try: + scope_class = self.scope_classes.get(scope) + scope_obj = scope_class(name=_id) + except Exception: + raise web.HTTPNotFound(reason="Scope object not found") # Check User permissions on called scope if not scope_obj.has_scope_permission(request.get("user"), "update"): raise web.HTTPForbidden(reason="User does not have Scope update permission.") - try: - # Add/Update Scope data in DB. Optimize set's and delete's - - # Validate 'key' param - dict_key = "**" - if data.get("key", None): - if not isinstance(data.get("key", None), dict): - raise ValueError("Invalid key format. Must be json type.") - dict_key = data.get("key") - Helpers.replace_dict_values(dict_key, "*", "**") - - # New Scope Data (dict) - if not dict_key == "**": - new_dict = Helpers.update_dict(dict_key, data.get("data", {})) - else: - new_dict = data.get("data", {}) - - # track scope changes - new_dict.update(self.track_scope(request, scope)) - - # Stored Scope Data (dict) - try: - movai_db = MovaiDB() - old_dict = movai_db.get({scope: {_id: dict_key}}).get(scope).get(_id) - except AttributeError: - old_dict = {} - - pipe = movai_db.create_pipe() - - deleted = [] - scope_updates = scope_obj.calc_scope_update(old_dict, new_dict) - for scope_obj in scope_updates: - to_delete = scope_obj.get("to_delete") - if to_delete: - if list(to_delete.keys())[0] == "PortsInst" and scope == "Node": - port_name = list(to_delete["PortsInst"].keys())[0] - if port_name not in deleted: - # in case we are deleting a Port from node, then use the regular delete - # in order to delete the exposedPorts from flows - Node(_id).delete("PortsInst", port_name) - deleted.append(port_name) - movai_db.unsafe_delete({scope: {_id: to_delete}}, pipe=pipe) - - to_set = scope_obj.get("to_set") - if to_set: - movai_db.set({scope: {_id: to_set}}, pipe=pipe) - - # Execute + if scope.lower() == "callback": + scope_obj.__dict__.update(self.track_scope(request, scope)) + scope_obj.save() resp = True - if scope_updates: - resp = bool(movai_db.execute_pipe(pipe)) - - # Store scope_updates on the request to use in middleware - request["scope_updates"] = scope_updates - - except Exception as exc: - # an object was created but there was an error - # object must be deleted - if obj_created: - movai_db.unsafe_delete({scope: {_id: "*"}}) - raise web.HTTPBadRequest(reason=str(exc)) + else: + try: + # Add/Update Scope data in DB. Optimize set's and delete's + + # Validate 'key' param + dict_key = "**" + if data.get("key", None): + if not isinstance(data.get("key", None), dict): + raise ValueError("Invalid key format. Must be json type.") + dict_key = data.get("key") + Helpers.replace_dict_values(dict_key, "*", "**") + + # New Scope Data (dict) + if not dict_key == "**": + new_dict = Helpers.update_dict(dict_key, data.get("data", {})) + else: + new_dict = data.get("data", {}) + + # track scope changes + # update LastUpdate + new_dict.update(self.track_scope(request, scope)) + + # Stored Scope Data (dict) + try: + movai_db = MovaiDB() + old_dict = movai_db.get({scope: {_id: dict_key}}).get(scope).get(_id) + except AttributeError: + old_dict = {} + + pipe = movai_db.create_pipe() + + deleted = [] + scope_updates = scope_obj.calc_scope_update(old_dict, new_dict) + for scope_obj in scope_updates: + to_delete = scope_obj.get("to_delete") + if to_delete: + if list(to_delete.keys())[0] == "PortsInst" and scope == "Node": + port_name = list(to_delete["PortsInst"].keys())[0] + if port_name not in deleted: + # in case we are deleting a Port from node, then use the regular delete + # in order to delete the exposedPorts from flows + Node(_id).delete("PortsInst", port_name) + deleted.append(port_name) + movai_db.unsafe_delete({scope: {_id: to_delete}}, pipe=pipe) + + to_set = scope_obj.get("to_set") + if to_set: + movai_db.set({scope: {_id: to_set}}, pipe=pipe) + + # Execute + resp = True + if scope_updates: + resp = bool(movai_db.execute_pipe(pipe)) + + # Store scope_updates on the request to use in middleware + request["scope_updates"] = scope_updates + + except Exception as exc: + # an object was created but there was an error + # object must be deleted + if obj_created: + movai_db.unsafe_delete({scope: {_id: "*"}}) + raise web.HTTPBadRequest(reason=str(exc)) return web.json_response({"success": resp, "name": _id}, headers={"Server": "Movai-server"}) From a01670b45c129716125aeb1fbbac44bc0087ff67 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Thu, 8 Jun 2023 09:22:01 +0300 Subject: [PATCH 02/26] removed logs, added TODO --- backend/endpoints/api/v1/restapi.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 44c78287..c80c10e1 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -723,7 +723,6 @@ async def get_scope(self, request: web.Request) -> web.Response: scope = request.match_info.get("scope") _id = request.match_info.get("name", False) - LOGGER.warning("Calling get_scope with scope: %s and id: %s", scope, _id) if _id: try: scope_obj = self.scope_classes[scope](_id) @@ -737,22 +736,21 @@ async def get_scope(self, request: web.Request) -> web.Response: if scope.lower() in ["callback", "node"]: if scope.lower() == "callback": + # TODO change this key = f"Movai:Callback:{_id}:__UNVERSIONED__" if key in cache: - LOGGER.error(f" ############## retreiving from cache {key}") scope_obj = cache[key] else: scope_obj = Callback(_id) - LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") cache[key] = scope_obj elif scope.lower() == "node": from dal.new_models import Node + # TODO change this key = f"Movai:Node:{_id}:__UNVERSIONED__" if key in cache: scope_obj = cache[key] else: scope_obj = Node(_id) - LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") cache[key] = scope_obj scope_result = scope_obj.dict() From 5b73092d830686e845f584805424f2fb6322bce2 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 13 Jun 2023 11:44:24 +0300 Subject: [PATCH 03/26] moved cache into BaseModel builtin --- backend/endpoints/api/v1/restapi.py | 18 +++--------------- backend/endpoints/static.py | 3 ++- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index c80c10e1..0964a756 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -37,7 +37,7 @@ from dal.movaidb import MovaiDB from dal.scopes.application import Application #from dal.scopes.callback import Callback -from dal.new_models import Callback, cache +from dal.new_models import Callback from dal.scopes.configuration import Configuration from dal.scopes.flow import Flow from dal.scopes.form import Form @@ -736,22 +736,10 @@ async def get_scope(self, request: web.Request) -> web.Response: if scope.lower() in ["callback", "node"]: if scope.lower() == "callback": - # TODO change this - key = f"Movai:Callback:{_id}:__UNVERSIONED__" - if key in cache: - scope_obj = cache[key] - else: - scope_obj = Callback(_id) - cache[key] = scope_obj + scope_obj = Callback(_id) elif scope.lower() == "node": from dal.new_models import Node - # TODO change this - key = f"Movai:Node:{_id}:__UNVERSIONED__" - if key in cache: - scope_obj = cache[key] - else: - scope_obj = Node(_id) - cache[key] = scope_obj + scope_obj = Node(_id) scope_result = scope_obj.dict() else: diff --git a/backend/endpoints/static.py b/backend/endpoints/static.py index 6e308ff7..ad03f7fd 100644 --- a/backend/endpoints/static.py +++ b/backend/endpoints/static.py @@ -91,12 +91,13 @@ async def get_static_file(self, request: web.Request) -> web.Response: package_name = request.match_info["package_name"] package_file = request.match_info["package_file"] + # get file from redis output = await asyncio.get_event_loop().run_in_executor( None, self._fetch_file_from_redis, package_name, package_file ) - if output is None: + if not output: raise web.HTTPNotFound() # guess content type From 9f68628f95ffff8564537b90e011d09f3b33b4c1 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 20 Jun 2023 13:16:46 +0300 Subject: [PATCH 04/26] used Configuration. --- backend/endpoints/api/v1/restapi.py | 32 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 0964a756..23e51a28 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -38,11 +38,11 @@ from dal.scopes.application import Application #from dal.scopes.callback import Callback from dal.new_models import Callback -from dal.scopes.configuration import Configuration +from dal.new_models import Configuration +from dal.new_models import Node from dal.scopes.flow import Flow from dal.scopes.form import Form from dal.scopes.message import Message -from dal.scopes.node import Node from dal.scopes.package import Package from dal.scopes.ports import Ports from dal.scopes.robot import Robot @@ -355,8 +355,7 @@ def get_spa_configuration(self, application): for cfg_name in configs: try: obj = Configuration(cfg_name) - _json = dict(yaml.safe_load(obj.Yaml) or {}) - output.update(_json) + output.update(obj.data) except Exception as error: LOGGER.info(str(error)) @@ -734,12 +733,14 @@ async def get_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node"]: + if scope.lower() in ["callback", "node", "configuration"]: if scope.lower() == "callback": scope_obj = Callback(_id) elif scope.lower() == "node": from dal.new_models import Node scope_obj = Node(_id) + else: + scope_obj = Configuration(_id) scope_result = scope_obj.dict() else: @@ -757,12 +758,14 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node"]: + if scope.lower() in ["callback", "node", "configuration"]: if scope == "Callback": objs = Callback.select() - else: + elif scope == "Node": from dal.new_models import Node objs = Node.select() + else: + objs = Configuration.select() LOGGER.error(f"importing from new models {objs}") scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} LOGGER.error(scope_result) @@ -912,6 +915,10 @@ async def post_to_scope(self, request: web.Request) -> web.Response: try: if scope.lower() == "callback": scope_obj = Callback(**{"Callback": {label: data["data"]}}) + elif scope.lower() == "node": + scope_obj = Node(**{"Node": {label: data["data"]}}) + elif scope.lower() == "configuration": + scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) else: label = data["data"].get("Label") scope_class = self.scope_classes.get(scope) @@ -930,6 +937,15 @@ async def post_to_scope(self, request: web.Request) -> web.Response: Callback(_id) label = data["data"].get("Label") scope_obj = Callback(**{"Callback": {label: data["data"]}}) + elif scope.lower() == "node": + from dal.new_models import Node + Node(_id) + label = data["data"].get("Label") + scope_obj = Node(**{"Node": {label: data["data"]}}) + elif scope.lower() == "configuration": + Configuration(_id) + label = data["data"].get("Label") + scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) else: # Check if scope exists try: @@ -942,7 +958,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "update"): raise web.HTTPForbidden(reason="User does not have Scope update permission.") - if scope.lower() == "callback": + if scope.lower() in ["callback", "node", "configuration"]: scope_obj.__dict__.update(self.track_scope(request, scope)) scope_obj.save() resp = True From edce3948f43c4ece76a537397659999aaf558f00 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Sun, 2 Jul 2023 11:19:30 +0300 Subject: [PATCH 05/26] using the new flow pydantic --- backend/endpoints/api/v1/restapi.py | 53 +++++++---------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 23e51a28..ae92e141 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -40,7 +40,7 @@ from dal.new_models import Callback from dal.new_models import Configuration from dal.new_models import Node -from dal.scopes.flow import Flow +from dal.new_models import Flow from dal.scopes.form import Form from dal.scopes.message import Message from dal.scopes.package import Package @@ -48,7 +48,7 @@ from dal.scopes.robot import Robot from dal.scopes.statemachine import StateMachine from dal.scopes.user import User - +from dal.new_models import MovaiBaseModel try: from movai_core_enterprise.message_client_handlers.metrics import Metrics from movai_core_enterprise.scopes.annotation import Annotation @@ -733,15 +733,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node", "configuration"]: - if scope.lower() == "callback": - scope_obj = Callback(_id) - elif scope.lower() == "node": - from dal.new_models import Node - scope_obj = Node(_id) - else: - scope_obj = Configuration(_id) - + if issubclass(self.scope_classes[scope], MovaiBaseModel): scope_result = scope_obj.dict() else: scope_result = MovaiDB().get({scope: {_id: "**"}}) @@ -758,17 +750,10 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node", "configuration"]: - if scope == "Callback": - objs = Callback.select() - elif scope == "Node": - from dal.new_models import Node - objs = Node.select() - else: - objs = Configuration.select() - LOGGER.error(f"importing from new models {objs}") + if issubclass(self.scope_classes[scope], MovaiBaseModel): + objs = self.scope_classes[scope].select() + scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} - LOGGER.error(scope_result) scope_result = {scope: scope_result} else: scope_result = MovaiDB().get_by_args(scope) @@ -913,12 +898,8 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: - if scope.lower() == "callback": - scope_obj = Callback(**{"Callback": {label: data["data"]}}) - elif scope.lower() == "node": - scope_obj = Node(**{"Node": {label: data["data"]}}) - elif scope.lower() == "configuration": - scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) + if issubclass(self.scope_classes[scope], MovaiBaseModel): + scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: label = data["data"].get("Label") scope_class = self.scope_classes.get(scope) @@ -932,20 +913,11 @@ async def post_to_scope(self, request: web.Request) -> web.Response: except Exception: raise web.HTTPBadRequest(reason="This already exists") else: - if scope.lower() == "callback": + if issubclass(self.scope_classes[scope], MovaiBaseModel): # check if exist - Callback(_id) - label = data["data"].get("Label") - scope_obj = Callback(**{"Callback": {label: data["data"]}}) - elif scope.lower() == "node": - from dal.new_models import Node - Node(_id) + self.scope_classes[scope](_id) label = data["data"].get("Label") - scope_obj = Node(**{"Node": {label: data["data"]}}) - elif scope.lower() == "configuration": - Configuration(_id) - label = data["data"].get("Label") - scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) + scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: # Check if scope exists try: @@ -958,7 +930,8 @@ async def post_to_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "update"): raise web.HTTPForbidden(reason="User does not have Scope update permission.") - if scope.lower() in ["callback", "node", "configuration"]: + + if issubclass(self.scope_classes[scope], MovaiBaseModel): scope_obj.__dict__.update(self.track_scope(request, scope)) scope_obj.save() resp = True From 40fee803f9761b538b84c1ef128fc48b67e50842 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Thu, 13 Jul 2023 09:33:49 +0300 Subject: [PATCH 06/26] used pydantic instead of MovaiBaseModel --- backend/endpoints/api/v1/restapi.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index ae92e141..124e117e 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -21,7 +21,7 @@ from string import Template from urllib.parse import unquote from typing import Any -import yaml +import pydantic from aiohttp import web @@ -48,7 +48,7 @@ from dal.scopes.robot import Robot from dal.scopes.statemachine import StateMachine from dal.scopes.user import User -from dal.new_models import MovaiBaseModel + try: from movai_core_enterprise.message_client_handlers.metrics import Metrics from movai_core_enterprise.scopes.annotation import Annotation @@ -733,7 +733,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_result = scope_obj.dict() else: scope_result = MovaiDB().get({scope: {_id: "**"}}) @@ -750,7 +750,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): objs = self.scope_classes[scope].select() scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} @@ -898,7 +898,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: label = data["data"].get("Label") @@ -913,7 +913,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: except Exception: raise web.HTTPBadRequest(reason="This already exists") else: - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): # check if exist self.scope_classes[scope](_id) label = data["data"].get("Label") @@ -931,7 +931,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPForbidden(reason="User does not have Scope update permission.") - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_obj.__dict__.update(self.track_scope(request, scope)) scope_obj.save() resp = True From c0ef840fe8b18be3578a306310f272f082a661de Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 6 Jun 2023 16:52:37 +0300 Subject: [PATCH 07/26] used new models --- backend/endpoints/api/v1/restapi.py | 175 ++++++++++++++++++---------- 1 file changed, 113 insertions(+), 62 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index ccea246d..0fb0f797 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -36,7 +36,8 @@ from dal.models.role import Role from dal.movaidb import MovaiDB from dal.scopes.application import Application -from dal.scopes.callback import Callback +#from dal.scopes.callback import Callback +from dal.new_models import Callback, cache from dal.scopes.configuration import Configuration from dal.scopes.flow import Flow from dal.scopes.form import Form @@ -125,7 +126,8 @@ async def cloud_func(self, request): callback = GD_Callback(callback_name, self.node_name, "cloud", False) # Check User permissions - scope_obj = self.scope_classes["Callback"](name=callback_name) + #scope_obj = self.scope_classes["Callback"](name=callback_name) + scope_obj = Callback(callback_name) if not scope_obj.has_permission(request.get("user"), "execute", app_name): raise ValueError("User does not have permission") @@ -699,17 +701,41 @@ async def get_scope(self, request: web.Request) -> web.Response: scope = request.match_info.get("scope") _id = request.match_info.get("name", False) + LOGGER.warning("Calling get_scope with scope: %s and id: %s", scope, _id) if _id: try: scope_obj = self.scope_classes[scope](_id) - except Exception: + except Exception as e: + LOGGER.error(f"caught error while creating scope object, exception: {e}") raise web.HTTPNotFound(reason="Required scope not found.") # Check User permissions if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - scope_result = MovaiDB().get({scope: {_id: "**"}}) + if scope.lower() in ["callback", "node"]: + if scope.lower() == "callback": + key = f"Movai:Callback:{_id}:__UNVERSIONED__" + if key in cache: + LOGGER.error(f" ############## retreiving from cache {key}") + scope_obj = cache[key] + else: + scope_obj = Callback(_id) + LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") + cache[key] = scope_obj + elif scope.lower() == "node": + from dal.new_models import Node + key = f"Movai:Node:{_id}:__UNVERSIONED__" + if key in cache: + scope_obj = cache[key] + else: + scope_obj = Node(_id) + LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") + cache[key] = scope_obj + + scope_result = scope_obj.dict() + else: + scope_result = MovaiDB().get({scope: {_id: "**"}}) result = scope_result[scope][_id] # If Scope User add permissions list @@ -723,7 +749,18 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - scope_result = MovaiDB().get_by_args(scope) + if scope.lower() in ["callback", "node"]: + if scope == "Callback": + objs = Callback.select() + else: + from dal.new_models import Node + objs = Node.select() + LOGGER.error(f"importing from new models {objs}") + scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} + LOGGER.error(scope_result) + scope_result = {scope: scope_result} + else: + scope_result = MovaiDB().get_by_args(scope) result = scope_result.get(scope, {}) if not result: @@ -865,58 +902,72 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: - label = data["data"].get("Label") - scope_class = self.scope_classes.get(scope) - struct = scope_class(label, new=True) - struct.Label = ( - label # just for now, this wont be needed when we merge branch "labeling" - ) - _id = struct.name - obj_created = _id - - scope_obj = scope_class(name=_id) + if scope.lower() == "callback": + scope_obj = Callback(**{"Callback": {label: data["data"]}}) + else: + label = data["data"].get("Label") + scope_class = self.scope_classes.get(scope) + struct = scope_class(label, new=True) + struct.Label = ( + label # just for now, this wont be needed when we merge branch "labeling" + ) + _id = struct.name + obj_created = _id + scope_obj = scope_class(name=_id) except Exception: raise web.HTTPBadRequest(reason="This already exists") else: - # Check if scope exists - try: - scope_class = self.scope_classes.get(scope) - scope_obj = scope_class(name=_id) - except Exception: - raise web.HTTPNotFound(reason="Scope object not found") + if scope.lower() == "callback": + # check if exist + Callback(_id) + label = data["data"].get("Label") + scope_obj = Callback(**{"Callback": {label: data["data"]}}) + else: + # Check if scope exists + try: + scope_class = self.scope_classes.get(scope) + scope_obj = scope_class(name=_id) + except Exception: + raise web.HTTPNotFound(reason="Scope object not found") # Check User permissions on called scope if not scope_obj.has_scope_permission(request.get("user"), "update"): raise web.HTTPForbidden(reason="User does not have Scope update permission.") - try: - # Add/Update Scope data in DB. Optimize set's and delete's - - # Validate 'key' param - dict_key = "**" - if data.get("key", None): - if not isinstance(data.get("key", None), dict): - raise ValueError("Invalid key format. Must be json type.") - dict_key = data.get("key") - Helpers.replace_dict_values(dict_key, "*", "**") - - # New Scope Data (dict) - if not dict_key == "**": - new_dict = Helpers.update_dict(dict_key, data.get("data", {})) - else: - new_dict = data.get("data", {}) - - # track scope changes - new_dict.update(self.track_scope(request, scope)) - - # Stored Scope Data (dict) + if scope.lower() == "callback": + scope_obj.__dict__.update(self.track_scope(request, scope)) + scope_obj.save() + resp = True + else: try: - movai_db = MovaiDB() - old_dict = movai_db.get({scope: {_id: dict_key}}).get(scope).get(_id) - except AttributeError: - old_dict = {} + # Add/Update Scope data in DB. Optimize set's and delete's + + # Validate 'key' param + dict_key = "**" + if data.get("key", None): + if not isinstance(data.get("key", None), dict): + raise ValueError("Invalid key format. Must be json type.") + dict_key = data.get("key") + Helpers.replace_dict_values(dict_key, "*", "**") + + # New Scope Data (dict) + if not dict_key == "**": + new_dict = Helpers.update_dict(dict_key, data.get("data", {})) + else: + new_dict = data.get("data", {}) + + # track scope changes + # update LastUpdate + new_dict.update(self.track_scope(request, scope)) + + # Stored Scope Data (dict) + try: + movai_db = MovaiDB() + old_dict = movai_db.get({scope: {_id: dict_key}}).get(scope).get(_id) + except AttributeError: + old_dict = {} - pipe = movai_db.create_pipe() + pipe = movai_db.create_pipe() ports_deleted = [] scope_updates = scope_obj.calc_scope_update(old_dict, new_dict) @@ -936,24 +987,24 @@ async def post_to_scope(self, request: web.Request) -> web.Response: ports_deleted.append(port_name) movai_db.unsafe_delete({scope: {_id: {key: value}}}, pipe=pipe) - to_set = scope_obj.get("to_set") - if to_set: - movai_db.set({scope: {_id: to_set}}, pipe=pipe) + to_set = scope_obj.get("to_set") + if to_set: + movai_db.set({scope: {_id: to_set}}, pipe=pipe) - # Execute - resp = True - if scope_updates: - resp = bool(movai_db.execute_pipe(pipe)) + # Execute + resp = True + if scope_updates: + resp = bool(movai_db.execute_pipe(pipe)) - # Store scope_updates on the request to use in middleware - request["scope_updates"] = scope_updates + # Store scope_updates on the request to use in middleware + request["scope_updates"] = scope_updates - except Exception as exc: - # an object was created but there was an error - # object must be deleted - if obj_created: - movai_db.unsafe_delete({scope: {_id: "*"}}) - raise web.HTTPBadRequest(reason=str(exc)) + except Exception as exc: + # an object was created but there was an error + # object must be deleted + if obj_created: + movai_db.unsafe_delete({scope: {_id: "*"}}) + raise web.HTTPBadRequest(reason=str(exc)) return web.json_response({"success": resp, "name": _id}, headers={"Server": "Movai-server"}) From 375774799281c88f609c51dd5db69c2e67a44f90 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Thu, 8 Jun 2023 09:22:01 +0300 Subject: [PATCH 08/26] removed logs, added TODO --- backend/endpoints/api/v1/restapi.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 0fb0f797..74f3c2ca 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -701,7 +701,6 @@ async def get_scope(self, request: web.Request) -> web.Response: scope = request.match_info.get("scope") _id = request.match_info.get("name", False) - LOGGER.warning("Calling get_scope with scope: %s and id: %s", scope, _id) if _id: try: scope_obj = self.scope_classes[scope](_id) @@ -715,22 +714,21 @@ async def get_scope(self, request: web.Request) -> web.Response: if scope.lower() in ["callback", "node"]: if scope.lower() == "callback": + # TODO change this key = f"Movai:Callback:{_id}:__UNVERSIONED__" if key in cache: - LOGGER.error(f" ############## retreiving from cache {key}") scope_obj = cache[key] else: scope_obj = Callback(_id) - LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") cache[key] = scope_obj elif scope.lower() == "node": from dal.new_models import Node + # TODO change this key = f"Movai:Node:{_id}:__UNVERSIONED__" if key in cache: scope_obj = cache[key] else: scope_obj = Node(_id) - LOGGER.error(f" %%%%%%%%%%%%%%%% Adding scope to cache {key}") cache[key] = scope_obj scope_result = scope_obj.dict() From 15a2f1adb12b83e535342381c5cdc4b71cb7a3e6 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 13 Jun 2023 11:44:24 +0300 Subject: [PATCH 09/26] moved cache into BaseModel builtin --- backend/endpoints/api/v1/restapi.py | 18 +++--------------- backend/endpoints/static.py | 1 + 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 74f3c2ca..8af0c241 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -37,7 +37,7 @@ from dal.movaidb import MovaiDB from dal.scopes.application import Application #from dal.scopes.callback import Callback -from dal.new_models import Callback, cache +from dal.new_models import Callback from dal.scopes.configuration import Configuration from dal.scopes.flow import Flow from dal.scopes.form import Form @@ -714,22 +714,10 @@ async def get_scope(self, request: web.Request) -> web.Response: if scope.lower() in ["callback", "node"]: if scope.lower() == "callback": - # TODO change this - key = f"Movai:Callback:{_id}:__UNVERSIONED__" - if key in cache: - scope_obj = cache[key] - else: - scope_obj = Callback(_id) - cache[key] = scope_obj + scope_obj = Callback(_id) elif scope.lower() == "node": from dal.new_models import Node - # TODO change this - key = f"Movai:Node:{_id}:__UNVERSIONED__" - if key in cache: - scope_obj = cache[key] - else: - scope_obj = Node(_id) - cache[key] = scope_obj + scope_obj = Node(_id) scope_result = scope_obj.dict() else: diff --git a/backend/endpoints/static.py b/backend/endpoints/static.py index 1d8e03d5..743ada31 100644 --- a/backend/endpoints/static.py +++ b/backend/endpoints/static.py @@ -91,6 +91,7 @@ async def get_static_file(self, request: web.Request) -> web.Response: package_name = request.match_info["package_name"] package_file = request.match_info["package_file"] + # get file from redis output = await asyncio.get_event_loop().run_in_executor( None, self._fetch_file_from_redis, package_name, package_file From 211f064b3f0802807316035688635a452ae73f6e Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 20 Jun 2023 13:16:46 +0300 Subject: [PATCH 10/26] used Configuration. --- backend/endpoints/api/v1/restapi.py | 32 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 8af0c241..63d6ca00 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -38,11 +38,11 @@ from dal.scopes.application import Application #from dal.scopes.callback import Callback from dal.new_models import Callback -from dal.scopes.configuration import Configuration +from dal.new_models import Configuration +from dal.new_models import Node from dal.scopes.flow import Flow from dal.scopes.form import Form from dal.scopes.message import Message -from dal.scopes.node import Node from dal.scopes.package import Package from dal.scopes.ports import Ports from dal.scopes.robot import Robot @@ -333,8 +333,7 @@ def get_spa_configuration(self, application): for cfg_name in configs: try: obj = Configuration(cfg_name) - _json = dict(yaml.safe_load(obj.Yaml) or {}) - output.update(_json) + output.update(obj.data) except Exception as error: LOGGER.info(str(error)) @@ -712,12 +711,14 @@ async def get_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node"]: + if scope.lower() in ["callback", "node", "configuration"]: if scope.lower() == "callback": scope_obj = Callback(_id) elif scope.lower() == "node": from dal.new_models import Node scope_obj = Node(_id) + else: + scope_obj = Configuration(_id) scope_result = scope_obj.dict() else: @@ -735,12 +736,14 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node"]: + if scope.lower() in ["callback", "node", "configuration"]: if scope == "Callback": objs = Callback.select() - else: + elif scope == "Node": from dal.new_models import Node objs = Node.select() + else: + objs = Configuration.select() LOGGER.error(f"importing from new models {objs}") scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} LOGGER.error(scope_result) @@ -890,6 +893,10 @@ async def post_to_scope(self, request: web.Request) -> web.Response: try: if scope.lower() == "callback": scope_obj = Callback(**{"Callback": {label: data["data"]}}) + elif scope.lower() == "node": + scope_obj = Node(**{"Node": {label: data["data"]}}) + elif scope.lower() == "configuration": + scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) else: label = data["data"].get("Label") scope_class = self.scope_classes.get(scope) @@ -908,6 +915,15 @@ async def post_to_scope(self, request: web.Request) -> web.Response: Callback(_id) label = data["data"].get("Label") scope_obj = Callback(**{"Callback": {label: data["data"]}}) + elif scope.lower() == "node": + from dal.new_models import Node + Node(_id) + label = data["data"].get("Label") + scope_obj = Node(**{"Node": {label: data["data"]}}) + elif scope.lower() == "configuration": + Configuration(_id) + label = data["data"].get("Label") + scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) else: # Check if scope exists try: @@ -920,7 +936,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "update"): raise web.HTTPForbidden(reason="User does not have Scope update permission.") - if scope.lower() == "callback": + if scope.lower() in ["callback", "node", "configuration"]: scope_obj.__dict__.update(self.track_scope(request, scope)) scope_obj.save() resp = True From b0be42a36450b3c0a4fe12d7d718991abdfa20cd Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Sun, 2 Jul 2023 11:19:30 +0300 Subject: [PATCH 11/26] using the new flow pydantic --- backend/endpoints/api/v1/restapi.py | 53 +++++++---------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 63d6ca00..9255cfa2 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -40,7 +40,7 @@ from dal.new_models import Callback from dal.new_models import Configuration from dal.new_models import Node -from dal.scopes.flow import Flow +from dal.new_models import Flow from dal.scopes.form import Form from dal.scopes.message import Message from dal.scopes.package import Package @@ -48,7 +48,7 @@ from dal.scopes.robot import Robot from dal.scopes.statemachine import StateMachine from dal.scopes.user import User - +from dal.new_models import MovaiBaseModel try: from movai_core_enterprise.message_client_handlers.metrics import Metrics from movai_core_enterprise.scopes.annotation import Annotation @@ -711,15 +711,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node", "configuration"]: - if scope.lower() == "callback": - scope_obj = Callback(_id) - elif scope.lower() == "node": - from dal.new_models import Node - scope_obj = Node(_id) - else: - scope_obj = Configuration(_id) - + if issubclass(self.scope_classes[scope], MovaiBaseModel): scope_result = scope_obj.dict() else: scope_result = MovaiDB().get({scope: {_id: "**"}}) @@ -736,17 +728,10 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if scope.lower() in ["callback", "node", "configuration"]: - if scope == "Callback": - objs = Callback.select() - elif scope == "Node": - from dal.new_models import Node - objs = Node.select() - else: - objs = Configuration.select() - LOGGER.error(f"importing from new models {objs}") + if issubclass(self.scope_classes[scope], MovaiBaseModel): + objs = self.scope_classes[scope].select() + scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} - LOGGER.error(scope_result) scope_result = {scope: scope_result} else: scope_result = MovaiDB().get_by_args(scope) @@ -891,12 +876,8 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: - if scope.lower() == "callback": - scope_obj = Callback(**{"Callback": {label: data["data"]}}) - elif scope.lower() == "node": - scope_obj = Node(**{"Node": {label: data["data"]}}) - elif scope.lower() == "configuration": - scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) + if issubclass(self.scope_classes[scope], MovaiBaseModel): + scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: label = data["data"].get("Label") scope_class = self.scope_classes.get(scope) @@ -910,20 +891,11 @@ async def post_to_scope(self, request: web.Request) -> web.Response: except Exception: raise web.HTTPBadRequest(reason="This already exists") else: - if scope.lower() == "callback": + if issubclass(self.scope_classes[scope], MovaiBaseModel): # check if exist - Callback(_id) - label = data["data"].get("Label") - scope_obj = Callback(**{"Callback": {label: data["data"]}}) - elif scope.lower() == "node": - from dal.new_models import Node - Node(_id) + self.scope_classes[scope](_id) label = data["data"].get("Label") - scope_obj = Node(**{"Node": {label: data["data"]}}) - elif scope.lower() == "configuration": - Configuration(_id) - label = data["data"].get("Label") - scope_obj = Configuration(**{"Configuration": {label: data["data"]}}) + scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: # Check if scope exists try: @@ -936,7 +908,8 @@ async def post_to_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "update"): raise web.HTTPForbidden(reason="User does not have Scope update permission.") - if scope.lower() in ["callback", "node", "configuration"]: + + if issubclass(self.scope_classes[scope], MovaiBaseModel): scope_obj.__dict__.update(self.track_scope(request, scope)) scope_obj.save() resp = True From 409af577056b9e812f5036bc40c0003689cf969e Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Thu, 13 Jul 2023 09:33:49 +0300 Subject: [PATCH 12/26] used pydantic instead of MovaiBaseModel --- backend/endpoints/api/v1/restapi.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 9255cfa2..59609cb6 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -21,7 +21,7 @@ from string import Template from urllib.parse import unquote from typing import Any -import yaml +import pydantic from aiohttp import web @@ -48,7 +48,7 @@ from dal.scopes.robot import Robot from dal.scopes.statemachine import StateMachine from dal.scopes.user import User -from dal.new_models import MovaiBaseModel + try: from movai_core_enterprise.message_client_handlers.metrics import Metrics from movai_core_enterprise.scopes.annotation import Annotation @@ -711,7 +711,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if not scope_obj.has_scope_permission(request.get("user"), "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_result = scope_obj.dict() else: scope_result = MovaiDB().get({scope: {_id: "**"}}) @@ -728,7 +728,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "read"): raise web.HTTPForbidden(reason="User does not have Scope permission.") - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): objs = self.scope_classes[scope].select() scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} @@ -876,7 +876,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: label = data["data"].get("Label") @@ -891,7 +891,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: except Exception: raise web.HTTPBadRequest(reason="This already exists") else: - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): # check if exist self.scope_classes[scope](_id) label = data["data"].get("Label") @@ -909,7 +909,7 @@ async def post_to_scope(self, request: web.Request) -> web.Response: raise web.HTTPForbidden(reason="User does not have Scope update permission.") - if issubclass(self.scope_classes[scope], MovaiBaseModel): + if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_obj.__dict__.update(self.track_scope(request, scope)) scope_obj.save() resp = True From c6f8f4d189fa6a304a0192831186239bb61b7006 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 24 Oct 2023 09:49:55 +0300 Subject: [PATCH 13/26] pydantic V2 changes --- backend/endpoints/api/v2/alerts.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/endpoints/api/v2/alerts.py b/backend/endpoints/api/v2/alerts.py index 5c2e8ab2..f064b52c 100644 --- a/backend/endpoints/api/v2/alerts.py +++ b/backend/endpoints/api/v2/alerts.py @@ -11,7 +11,7 @@ """ import json import re -from pydantic import Field, BaseModel, EmailStr, ValidationError +from pydantic import ConfigDict, Field, BaseModel, EmailStr, ValidationError from typing import List from aiohttp import web from backend.endpoints.api.v2.base import BaseWebApp @@ -25,9 +25,7 @@ class AlertsConfig(BaseModel): emails: List[EmailStr] = Field(default_factory=list) alerts: List[str] = Field(default_factory=list) - - class Config: - validate_assignment = True + model_config = ConfigDict(validate_assignment=True) class Meta: # global variables (like class variables) From f23853351a0208f174e6b1eb574ec857bb4f7a40 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Tue, 24 Oct 2023 12:01:45 +0300 Subject: [PATCH 14/26] pydantic V2 fixes --- backend/endpoints/api/v1/restapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 0c1b44de..169a8846 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -712,7 +712,7 @@ async def get_scope(self, request: web.Request) -> web.Response: raise web.HTTPForbidden(reason="User does not have Scope permission.") if issubclass(self.scope_classes[scope], pydantic.BaseModel ): - scope_result = scope_obj.dict() + scope_result = scope_obj.model_dump() else: scope_result = MovaiDB().get({scope: {_id: "**"}}) result = scope_result[scope][_id] @@ -731,7 +731,7 @@ async def get_scope(self, request: web.Request) -> web.Response: if issubclass(self.scope_classes[scope], pydantic.BaseModel ): objs = self.scope_classes[scope].select() - scope_result = {obj.name: obj.dict()["Callback"][obj.name] for obj in objs} + scope_result = {obj.name: obj.model_dump()["Callback"][obj.name] for obj in objs} scope_result = {scope: scope_result} else: scope_result = MovaiDB().get_by_args(scope) From f6239b23d47a0d31c59dfea545a8e1898ab867f8 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Thu, 26 Oct 2023 12:08:08 +0300 Subject: [PATCH 15/26] fixed error issue when creating scope --- backend/endpoints/api/v1/restapi.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 169a8846..bfa14595 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -36,7 +36,6 @@ from dal.models.role import Role from dal.movaidb import MovaiDB from dal.scopes.application import Application -#from dal.scopes.callback import Callback from dal.new_models import Callback from dal.new_models import Configuration from dal.new_models import Node @@ -872,14 +871,14 @@ async def post_to_scope(self, request: web.Request) -> web.Response: if not request.get("user").has_permission(scope, "create"): raise web.HTTPForbidden(reason="User does not have Scope create permission.") - if not data["data"].get("Label", None): + label = data["data"].get("Label", None) + if not label: raise web.HTTPBadRequest(reason="Label is required to create new scope") try: if issubclass(self.scope_classes[scope], pydantic.BaseModel ): scope_obj = self.scope_classes[scope](**{scope: {label: data["data"]}}) else: - label = data["data"].get("Label") scope_class = self.scope_classes.get(scope) struct = scope_class(label, new=True) struct.Label = ( From 7551dad18d633f3edd86911d3516810dcae68352 Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Thu, 26 Oct 2023 13:16:05 +0300 Subject: [PATCH 16/26] update imports --- backend/endpoints/api/v1/restapi.py | 2 +- backend/tools/deploy_app.py | 16 +++++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index bfa14595..000104c5 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -50,7 +50,7 @@ try: from movai_core_enterprise.message_client_handlers.metrics import Metrics - from movai_core_enterprise.scopes.annotation import Annotation + from movai_core_enterprise.new_models import Annotation from movai_core_enterprise.scopes.graphicscene import GraphicScene from movai_core_enterprise.scopes.layout import Layout from movai_core_enterprise.scopes.shareddatatemplate import SharedDataTemplate diff --git a/backend/tools/deploy_app.py b/backend/tools/deploy_app.py index f1623113..99546b7b 100644 --- a/backend/tools/deploy_app.py +++ b/backend/tools/deploy_app.py @@ -12,7 +12,7 @@ import argparse import json import os -from dal.scopes.application import Application +from dal.new_models import Application JSON_FILE = "package.json" @@ -85,15 +85,8 @@ def deploy(args): # don't generate metadata return - app = None - - try: - app = Application(app_json["name"]) - print(f"Updating application {app.name}") - - except Exception: - app = Application(app_json["name"], new=True) - print(f"Creating application {app.name}") + app = Application.model_validate(app_json["name"]) + print(f"Updating application {app.name}") print(f"-" * 100) @@ -117,7 +110,8 @@ def deploy(args): except AttributeError: print(f"Attribute {key} does not exist") + app.save() -if __name__ == "__main__": +if __name__ == "__main__": main() From 072da886aeafe70f37035cd9413ae1936b4ef1fb Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Wed, 1 Nov 2023 10:15:27 +0200 Subject: [PATCH 17/26] circular import errors, added role --- backend/core/permission.py | 4 +-- backend/endpoints/api/v1/restapi.py | 39 ++++++++++++++--------------- backend/endpoints/api/v2/base.py | 9 +++---- backend/endpoints/api/v2/role.py | 15 +++++------ backend/endpoints/static.py | 11 +++----- backend/tools/new_user.py | 2 +- backend/tools/upload_ui.py | 17 +++++-------- backend/tools/user_tool.py | 2 +- 8 files changed, 44 insertions(+), 55 deletions(-) diff --git a/backend/core/permission.py b/backend/core/permission.py index 7e854893..4695c391 100644 --- a/backend/core/permission.py +++ b/backend/core/permission.py @@ -14,7 +14,7 @@ from movai_core_shared.logger import Log from dal.models.scopestree import scopes -from dal.models.role import Role +from dal.new_models import Role LOGGER = Log.get_logger(__name__) @@ -54,7 +54,7 @@ def _init_structure(cls, roles: List[str] = []) -> None: roles = Role.list_roles_names() for role_name in roles: try: - role_obj = scopes.from_path(role_name, scope="Role") + role_obj = Role(role_name) for (resource_key, permissions) in role_obj.Resources.items(): for permission in permissions: if permission: diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 000104c5..06cc06c0 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -20,7 +20,7 @@ from mimetypes import guess_type from string import Template from urllib.parse import unquote -from typing import Any +from typing import Any, List import pydantic from aiohttp import web @@ -33,17 +33,17 @@ from dal.models.acl import NewACLManager from dal.models.lock import Lock from dal.models.var import Var -from dal.models.role import Role +from dal.new_models.role import Role from dal.movaidb import MovaiDB -from dal.scopes.application import Application +from dal.new_models import Application from dal.new_models import Callback from dal.new_models import Configuration from dal.new_models import Node from dal.new_models import Flow from dal.scopes.form import Form -from dal.scopes.message import Message -from dal.scopes.package import Package -from dal.scopes.ports import Ports +from dal.new_models import Message +from dal.new_models import Package +from dal.new_models import Ports from dal.scopes.robot import Robot from dal.scopes.statemachine import StateMachine from dal.scopes.user import User @@ -287,7 +287,9 @@ async def get_spa(self, request): # Get app information app = Application(app_name) content_type = guess_type(app.EntryPoint)[0] - html = Package(app.Package).File[app.EntryPoint].Value + # html = Package(app.Package).File[app.EntryPoint].Value + html = Package(app.Package).get_value(app.EntryPoint) + html = self.spa_parse_template(app, html, request) except Exception as error: @@ -297,7 +299,7 @@ async def get_spa(self, request): body=html, content_type=content_type, headers={"Server": "Movai-server"} ) - def spa_parse_template(self, application, html, request): + def spa_parse_template(self, application: Application, html, request): """parse application params""" serverdata = {"pathname": f"{self.api_version}apps/{application.name}/"} @@ -306,7 +308,7 @@ def spa_parse_template(self, application, html, request): serverdata.update(self.get_spa_configuration(application)) # get application meta-data serverdata.update( - {"Application": application.get_dict()["Application"][application.name]} + {"Application": application.model_dump()["Application"][application.name]} ) except Exception as error: LOGGER.error(str(error)) @@ -324,7 +326,7 @@ def spa_parse_template(self, application, html, request): } return Template(html.decode("utf-8")).safe_substitute(**params) - def get_spa_configuration(self, application): + def get_spa_configuration(self, application: Application): """get default configuration and updated it with user custom configuration""" output = {} @@ -626,18 +628,15 @@ def create_application_format(url, label, icon, enable, app_type): try: permissions = NewACLManager.get_permissions()["Applications"] - scope = "Application" - scope_result = MovaiDB().get_by_args(scope) - application_raw_data = scope_result.get(scope, {}) output = {"success": True, "result": []} - for key in application_raw_data: - app = application_raw_data[key] - url = app["Package"] if app["Type"] == "application" else app["EntryPoint"] - label = app["Label"] - icon = app["Icon"] - enable = len(list(filter(lambda x: x == key, permissions))) > 0 - app_type = app["Type"] + apps: List[Application] = Application.select() + for app in apps: + url = app.Package if app.Type == "application" else app.EntryPoint + label = app.Label + icon = app.Icon + enable = len(list(filter(lambda x: x == app.name, permissions))) > 0 + app_type = app.Type output["result"].append( create_application_format(url, label, icon, enable, app_type) ) diff --git a/backend/endpoints/api/v2/base.py b/backend/endpoints/api/v2/base.py index a85fccb0..552f2780 100644 --- a/backend/endpoints/api/v2/base.py +++ b/backend/endpoints/api/v2/base.py @@ -27,15 +27,14 @@ from dal.movaidb import MovaiDB from dal.models.aclobject import AclObject from dal.models.ldapconfig import LdapConfig - -from dal.models.role import Role +import dal.new_models.role from dal.models.internaluser import InternalUser from dal.models.remoteuser import RemoteUser - from gd_node.protocols.http.middleware import redirect_not_found - from backend.http import IWebApp +Role = getattr(dal.new_models.role, "Role") + class RestBaseClass: """Base class for REST operations""" @@ -143,7 +142,7 @@ def validate_result(self, result: dict): @classmethod def validate_role(cls, roles: List[str]) -> None: if len(roles) == 0: - error_msg = f"Role must be specified for every user or group." + error_msg = "Role must be specified for every user or group." raise ValueError(error_msg) for role in roles: if not Role.is_exist(role): diff --git a/backend/endpoints/api/v2/role.py b/backend/endpoints/api/v2/role.py index 063445c0..31b08673 100644 --- a/backend/endpoints/api/v2/role.py +++ b/backend/endpoints/api/v2/role.py @@ -13,13 +13,15 @@ from typing import List from aiohttp import web from aiohttp.web_response import Response - -from dal.models.role import Role - from backend.http import WebAppManager from backend.endpoints.api.v2.base import BaseWebApp, RestBaseClass +def Role(): + import dal.new_models.role + return dal.new_models.role.Role + + class RoleRestBaseClass(RestBaseClass, ABC): """A Base class for Rest API of Role.""" @@ -30,8 +32,7 @@ def __init__(self) -> None: def extract_scope(self) -> None: """sets the scope the call is directed too.""" - self._scope_name = Role.__name__ - self._scope = self.scope_classes.get(self._scope_name) + self._scope = self.scope_classes.get("Role") @abstractmethod async def execute_imp(self) -> None: @@ -100,7 +101,7 @@ async def execute_imp(self) -> None: self.check_permissions() payload = await self._request.json() data = payload.get("data") - role_obj: Role = Role.create(data["Label"], data["Resources"]) + role_obj = Role().create(data["Label"], data["Resources"]) role_obj.update_time() self._result["success"] = True self._result["name"] = data["Label"] @@ -138,7 +139,7 @@ async def execute_imp(self) -> None: """This method deletes an Role object from the DB.""" self.extract_object() self.check_permissions() - Role.remove(self._object_name) + Role().remove(self._object_name) self._result["success"] = True self._result["name"] = self._object_name diff --git a/backend/endpoints/static.py b/backend/endpoints/static.py index 8785708f..9d02c930 100644 --- a/backend/endpoints/static.py +++ b/backend/endpoints/static.py @@ -21,7 +21,7 @@ import aiohttp_cors from aiohttp import web -from dal.movaidb import MovaiDB +from dal.new_models import Package from gd_node.protocols.http.middleware import ( save_node_type, @@ -76,13 +76,8 @@ def _fetch_file_from_redis(package_name: str, package_file: str) -> str: decoded_package_name = unquote(package_file) # use MovaiDB().get() increase performance - _file = MovaiDB().get( - {"Package": {package_name: {"File": {decoded_package_name: {"Value": "*"}}}}} - ) - try: - return _file["Package"][package_name]["File"][decoded_package_name]["Value"] - except KeyError: - return "" + _file = Package(package_name).File[decoded_package_name].Value + return _file async def get_static_file(self, request: web.Request) -> web.Response: """get static file from Package""" diff --git a/backend/tools/new_user.py b/backend/tools/new_user.py index fce5fad3..defb86a8 100644 --- a/backend/tools/new_user.py +++ b/backend/tools/new_user.py @@ -10,7 +10,7 @@ from dal.models.acl import NewACLManager from dal.models.internaluser import InternalUser -from dal.models.role import Role +from dal.new_models import Role LOGGER = Log.get_logger(__name__) diff --git a/backend/tools/upload_ui.py b/backend/tools/upload_ui.py index 4202cb09..b5ac4b5f 100644 --- a/backend/tools/upload_ui.py +++ b/backend/tools/upload_ui.py @@ -14,7 +14,7 @@ import tempfile import zipfile from movai_core_shared.logger import Log -from dal.scopes.package import Package +from dal.new_models import Package sys.path.append(os.path.abspath("..")) @@ -44,19 +44,14 @@ def main(build_folder: str, package_name: str): getFolderStructure(build_folder, build_files) - try: - pkg = Package(package_name) - pkg.remove() - del pkg - logger.info("Overwritting Package '%s'" % package_name) - except: - logger.info("Creating Package '%s'" % package_name) - - pkg = Package(package_name, new=True) + pkg = Package(package_name) + pkg.delete() + logger.info("Update Package '%s'" % package_name) for x in build_files: - pkg.add("File", x, Value=build_files[x]) + pkg.add_file(x, build_files[x]) logger.info("File '%s' added to package '%s'" % (x, package_name)) + pkg.save() if __name__ == "__main__": diff --git a/backend/tools/user_tool.py b/backend/tools/user_tool.py index c9037412..c8faf227 100755 --- a/backend/tools/user_tool.py +++ b/backend/tools/user_tool.py @@ -37,7 +37,7 @@ from movai_core_shared.logger import Log from dal.models.user import User -from dal.models.role import Role +from dal.new_models import Role from dal.models.internaluser import InternalUser CONVERT_COMMAND = "convert" From d7fab051aaaa8dbe3c11b2fd92dc8a28a2f6f6ee Mon Sep 17 00:00:00 2001 From: Moawiya Mograbi Date: Wed, 1 Nov 2023 18:10:57 +0200 Subject: [PATCH 18/26] Package fixes --- backend/endpoints/api/v1/restapi.py | 2 +- backend/endpoints/static.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 06cc06c0..9e8ff9d9 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -659,7 +659,7 @@ async def old_get_static_file(self, request: web.Request) -> web.Response: package_name = request.match_info["package_name"] package_file = request.match_info["package_file"] content_type = guess_type(package_file)[0] - output = Package(package_name).File[package_file].Value + output = Package(package_name).get_value(package_file) return web.Response( body=output, content_type=content_type, diff --git a/backend/endpoints/static.py b/backend/endpoints/static.py index 9d02c930..3e1cbec8 100644 --- a/backend/endpoints/static.py +++ b/backend/endpoints/static.py @@ -76,7 +76,7 @@ def _fetch_file_from_redis(package_name: str, package_file: str) -> str: decoded_package_name = unquote(package_file) # use MovaiDB().get() increase performance - _file = Package(package_name).File[decoded_package_name].Value + _file = Package(package_name).get_value(decoded_package_name) return _file async def get_static_file(self, request: web.Request) -> web.Response: From 9590aa1b77622e6ecdaaff3fd9fba882be19d3a3 Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Wed, 29 Nov 2023 23:50:49 +0200 Subject: [PATCH 19/26] replace select with find --- backend/endpoints/api/v1/restapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 9e8ff9d9..8ef77862 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -630,7 +630,7 @@ def create_application_format(url, label, icon, enable, app_type): permissions = NewACLManager.get_permissions()["Applications"] output = {"success": True, "result": []} - apps: List[Application] = Application.select() + apps: List[Application] = Application.find() for app in apps: url = app.Package if app.Type == "application" else app.EntryPoint label = app.Label @@ -727,7 +727,7 @@ async def get_scope(self, request: web.Request) -> web.Response: raise web.HTTPForbidden(reason="User does not have Scope permission.") if issubclass(self.scope_classes[scope], pydantic.BaseModel ): - objs = self.scope_classes[scope].select() + objs = self.scope_classes[scope].find() scope_result = {obj.name: obj.model_dump()["Callback"][obj.name] for obj in objs} scope_result = {scope: scope_result} From 990c00668d10e9c95eda6347c61ec872419ae595 Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Sun, 10 Dec 2023 09:23:49 +0200 Subject: [PATCH 20/26] replace find with get_all --- backend/endpoints/api/v1/restapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 075f50b9..e388e1a5 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -664,7 +664,7 @@ def create_application_format(url, label, icon, enable, app_type): permissions = NewACLManager.get_permissions()["Applications"] output = {"success": True, "result": []} - apps: List[Application] = Application.find() + apps: List[Application] = Application.get_all() for app in apps: url = app.Package if app.Type == "application" else app.EntryPoint label = app.Label @@ -761,7 +761,7 @@ async def get_scope(self, request: web.Request) -> web.Response: raise web.HTTPForbidden(reason="User does not have Scope permission.") if issubclass(self.scope_classes[scope], pydantic.BaseModel ): - objs = self.scope_classes[scope].find() + objs = self.scope_classes[scope].get_all() scope_result = {obj.name: obj.model_dump()["Callback"][obj.name] for obj in objs} scope_result = {scope: scope_result} From 8fbfed523c0d75c2ba1866bbafda07df918c5f43 Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Sun, 10 Dec 2023 23:53:29 +0200 Subject: [PATCH 21/26] update function name --- backend/endpoints/api/v1/restapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 19ad29f8..2ac44e0f 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -664,7 +664,7 @@ def create_application_format(url, label, icon, enable, app_type): permissions = NewACLManager.get_permissions()["Applications"] output = {"success": True, "result": []} - apps: List[Application] = Application.get_all() + apps: List[Application] = Application.get_all_models() for app in apps: url = app.Package if app.Type == "application" else app.EntryPoint label = app.Label @@ -761,7 +761,7 @@ async def get_scope(self, request: web.Request) -> web.Response: raise web.HTTPForbidden(reason="User does not have Scope permission.") if issubclass(self.scope_classes[scope], pydantic.BaseModel ): - objs = self.scope_classes[scope].get_all() + objs = self.scope_classes[scope].get_all_models() scope_result = {obj.name: obj.model_dump()["Callback"][obj.name] for obj in objs} scope_result = {scope: scope_result} From dc3c93b7707e0456c8e52c3efc4f1189566887b8 Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Sun, 10 Dec 2023 23:53:50 +0200 Subject: [PATCH 22/26] import objects from new_models --- .../v1/frontend/fleetmanager/statistics.py | 5 +- .../api/v1/frontend/ide/callbackeditor.py | 4 +- .../api/v1/frontend/ide/datavalidation.py | 6 +-- .../api/v1/frontend/ide/getportsdata.py | 2 +- .../endpoints/api/v1/frontend/ide/viewer.py | 47 +++++++++---------- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py b/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py index b71d6025..52b5cac9 100644 --- a/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py +++ b/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py @@ -29,9 +29,10 @@ from movai_core_shared.logger import Log from dal.models.var import Var -from dal.scopes.robot import Robot -from dal.scopes.configuration import Configuration +from dal.new_models.configuration import Configuration from dal.scopes.fleetrobot import FleetRobot +from dal.scopes.robot import Robot + LOGGER = Log.get_logger(__name__) diff --git a/backend/endpoints/api/v1/frontend/ide/callbackeditor.py b/backend/endpoints/api/v1/frontend/ide/callbackeditor.py index d930569c..61a475d0 100644 --- a/backend/endpoints/api/v1/frontend/ide/callbackeditor.py +++ b/backend/endpoints/api/v1/frontend/ide/callbackeditor.py @@ -3,8 +3,8 @@ Unauthorized copying of this file, via any medium is strictly prohibited Proprietary and confidential """ -from dal.models.callback import Callback -from dal.models.message import Message +from dal.new_models.callback import Callback +from dal.new_models.message import Message class CallbcakEditor: diff --git a/backend/endpoints/api/v1/frontend/ide/datavalidation.py b/backend/endpoints/api/v1/frontend/ide/datavalidation.py index af76b0ad..fca6ff05 100644 --- a/backend/endpoints/api/v1/frontend/ide/datavalidation.py +++ b/backend/endpoints/api/v1/frontend/ide/datavalidation.py @@ -8,6 +8,7 @@ from movai_core_shared.logger import Log from dal.models.scopestree import ScopesTree +from dal.new_models.configuration import Configuration LOGGER = Log.get_logger(__name__) @@ -32,7 +33,7 @@ def validate_configuration(config_string): config_key_path = config_string.replace("$(config ", "").replace(")", "").split(".") config_name = config_key_path[0] config_key_path.pop(0) - config = ScopesTree()().Configuration[config_name] + config = Configuration(config_name) val = config.get_value() for key in config_key_path: val = val[key] @@ -53,8 +54,7 @@ def validate_configuration_raw(value): config_key_path = value.split(".") config_name = config_key_path[0] config_key_path.pop(0) - scopes = ScopesTree() - config = scopes().Configuration[config_name] + config = Configuration(config_name) val = config.get_value() for key in config_key_path: val = val[key] diff --git a/backend/endpoints/api/v1/frontend/ide/getportsdata.py b/backend/endpoints/api/v1/frontend/ide/getportsdata.py index 12a16051..58584431 100644 --- a/backend/endpoints/api/v1/frontend/ide/getportsdata.py +++ b/backend/endpoints/api/v1/frontend/ide/getportsdata.py @@ -3,7 +3,7 @@ Unauthorized copying of this file, via any medium is strictly prohibited Proprietary and confidential """ -from dal.models.message import Message +from dal.new_models.message import Message def get_ports_data(): diff --git a/backend/endpoints/api/v1/frontend/ide/viewer.py b/backend/endpoints/api/v1/frontend/ide/viewer.py index 4a358d96..533ccd17 100644 --- a/backend/endpoints/api/v1/frontend/ide/viewer.py +++ b/backend/endpoints/api/v1/frontend/ide/viewer.py @@ -12,15 +12,16 @@ from movai_core_shared.logger import Log -from dal.models.scopestree import ScopesTree from dal.models.var import Var as Variable from dal.scopes.fleetrobot import FleetRobot -from dal.scopes.package import Package +from dal.new_models.package import Package LOGGER = Log.get_logger(__name__) try: - from movai_core_enterprise.scopes.graphicscene import GraphicScene + from movai_core_enterprise.new_models.annotation import Annotation + from movai_core_enterprise.models.graphicscene import GraphicScene as GraphicSceneModel + from movai_core_enterprise.scopes.graphicscene import GraphicScene as GraphicSceneScope except ImportError: LOGGER.warning("Failed to import GraphicScene, because movai_core_enterprise is not installed.") @@ -257,9 +258,9 @@ def add2scene_aux(tree_node, scene_name, tree_object, scene): def add2scene(tree_node, scene_name, tree_object): - scene_scope = ScopesTree().from_path(scene_name, scope="GraphicScene") - add2scene_aux(tree_node, scene_name, tree_object, scene_scope) - scene_scope.write() + scene = GraphicSceneModel(scene_name) + add2scene_aux(tree_node, scene_name, tree_object, scene) + scene.write() def add_annotation_to_object( @@ -309,13 +310,13 @@ def del2scene_aux(obj_name, scene_name, scene_scope): def del2scene(obj_name, scene_name): - scene_scope = ScopesTree().from_path(scene_name, scope="GraphicScene") - del2scene_aux(obj_name, scene_name, scene_scope) - scene_scope.write() + scene = GraphicSceneModel(scene_name) + del2scene_aux(obj_name, scene_name, scene) + scene.write() -def reset_scene(scene_path): - scene = ScopesTree().from_path(scene_path, scope="GraphicScene") +def reset_scene(scene_name): + scene = GraphicSceneModel(scene_name) types = list(filter(lambda x: x != "memory", list(scene.AssetType))) for asset_type in types: del scene.AssetType[asset_type] @@ -345,7 +346,7 @@ def __init__(self, scene_name): """Virtually private constructor.""" MemorySingleton.__instance = self self.scene_name = scene_name - self.memory = GraphicScene(scene_name).AssetType["memory"] + self.memory = GraphicSceneScope(scene_name).AssetType["memory"] def __set_tree(self, tree=[]): self.memory.add("AssetName", "tree", Value=tree) @@ -479,11 +480,10 @@ def delete_add_object(): delete_add_object() @classmethod - def on_retrieve_scene(cls, scene_path=DEFAULT_SCENE_NAME): - cls.migrate_poses_in_scene(scene_path) - my_scopes = ScopesTree() - scene = my_scopes.from_path(scene_path, scope="GraphicScene") - sprint("Scene types", scene_path, list(scene.AssetType)) + def on_retrieve_scene(cls, scene_name=DEFAULT_SCENE_NAME): + cls.migrate_poses_in_scene(scene_name) + scene = GraphicSceneModel(scene_name) + sprint("Scene types", scene_name, list(scene.AssetType)) return scene.AssetType["memory"].AssetName["tree"].Value.value @staticmethod @@ -513,16 +513,15 @@ def migrate_poses_in_scene(scene_path): try: types = ["KeyPoint", "Map", "Mesh", "GlobalRef", "Robot", "PointCloud", "NodeItem"] scene_name = scene_path.split("/")[-1] - scopes = ScopesTree() - new_scene = scopes().GraphicScene[scene_name] - scene = GraphicScene(scene_name) + scene_model = GraphicSceneModel(scene_name) + scene_scope = GraphicSceneScope(scene_name) for t in types: try: - for i in new_scene.AssetType[t].AssetName: - point = new_scene.AssetType[t].AssetName[i].Value + for i in scene_model.AssetType[t].AssetName: + point = scene_model.AssetType[t].AssetName[i].Value sprint("Migrate value", t, i) if isinstance(point._value, Pose): - scene.AssetType[t].AssetName[i].Value = point.value + scene_scope.AssetType[t].AssetName[i].Value = point.value except Exception as e: sprint("Caught exception while migrating types...", e) except Exception as e: @@ -537,7 +536,7 @@ def get_computed_annotations(annotations_name): computed_annotations_dict = {} for annotation_name in annotations_name: try: - annotation = ScopesTree().from_path(annotation_name, scope="Annotation") + annotation = Annotation(annotation_name) computed_annotations_dict.update(annotation.get_computed_annotation()) except Exception as e: sprint("Caught Exception while computing annotation", annotation_name, e) From 980fb347924e5ff24c48aa976c759176e602c844 Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Wed, 27 Dec 2023 11:39:12 +0200 Subject: [PATCH 23/26] revert back to Package from scopes --- backend/endpoints/api/v1/frontend/ide/viewer.py | 4 +++- backend/endpoints/api/v1/restapi.py | 9 ++++----- backend/endpoints/static.py | 6 ++++-- backend/tools/upload_ui.py | 4 +++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/endpoints/api/v1/frontend/ide/viewer.py b/backend/endpoints/api/v1/frontend/ide/viewer.py index 533ccd17..9525f3dc 100644 --- a/backend/endpoints/api/v1/frontend/ide/viewer.py +++ b/backend/endpoints/api/v1/frontend/ide/viewer.py @@ -13,8 +13,10 @@ from movai_core_shared.logger import Log from dal.models.var import Var as Variable + from dal.scopes.fleetrobot import FleetRobot -from dal.new_models.package import Package +from dal.scopes.package import Package + LOGGER = Log.get_logger(__name__) diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 2ac44e0f..4fe8d964 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -34,17 +34,17 @@ from dal.models.acl import NewACLManager from dal.models.lock import Lock from dal.models.var import Var -from dal.new_models.role import Role from dal.movaidb import MovaiDB from dal.new_models import Application from dal.new_models import Callback from dal.new_models import Configuration from dal.new_models import Node from dal.new_models import Flow -from dal.scopes.form import Form from dal.new_models import Message -from dal.new_models import Package from dal.new_models import Ports +from dal.new_models.role import Role +from dal.scopes.form import Form +from dal.scopes.package import Package from dal.scopes.robot import Robot from dal.scopes.statemachine import StateMachine from dal.scopes.user import User @@ -321,8 +321,7 @@ async def get_spa(self, request): # Get app information app = Application(app_name) content_type = guess_type(app.EntryPoint)[0] - # html = Package(app.Package).File[app.EntryPoint].Value - html = Package(app.Package).get_value(app.EntryPoint) + html = Package(app.Package).File[app.EntryPoint].Value html = self.spa_parse_template(app, html, request) diff --git a/backend/endpoints/static.py b/backend/endpoints/static.py index 3e1cbec8..e0ac8621 100644 --- a/backend/endpoints/static.py +++ b/backend/endpoints/static.py @@ -21,7 +21,7 @@ import aiohttp_cors from aiohttp import web -from dal.new_models import Package +from dal.scopes import Package from gd_node.protocols.http.middleware import ( save_node_type, @@ -76,7 +76,9 @@ def _fetch_file_from_redis(package_name: str, package_file: str) -> str: decoded_package_name = unquote(package_file) # use MovaiDB().get() increase performance - _file = Package(package_name).get_value(decoded_package_name) + + _file = Package(package_name).File[decoded_package_name].Value + return _file async def get_static_file(self, request: web.Request) -> web.Response: diff --git a/backend/tools/upload_ui.py b/backend/tools/upload_ui.py index b5ac4b5f..ecb37e54 100644 --- a/backend/tools/upload_ui.py +++ b/backend/tools/upload_ui.py @@ -13,8 +13,10 @@ import sys import tempfile import zipfile + from movai_core_shared.logger import Log -from dal.new_models import Package + +from dal.scopes import Package sys.path.append(os.path.abspath("..")) From 2dbf5bec0fd6b711ceda863b5710c5c15c90d8db Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Wed, 3 Jan 2024 20:00:05 +0200 Subject: [PATCH 24/26] code review fixes, revert to scopes annotation and role --- backend/core/permission.py | 2 +- .../v1/frontend/fleetmanager/statistics.py | 1 + .../api/v1/frontend/ide/datavalidation.py | 2 +- .../endpoints/api/v1/frontend/ide/viewer.py | 2 +- backend/endpoints/api/v1/restapi.py | 19 ++++++++++--------- backend/endpoints/api/v2/base.py | 8 ++++---- backend/endpoints/api/v2/role.py | 17 ++++++++--------- backend/tools/new_user.py | 2 +- backend/tools/user_tool.py | 2 +- 9 files changed, 28 insertions(+), 27 deletions(-) diff --git a/backend/core/permission.py b/backend/core/permission.py index 4695c391..c509a60b 100644 --- a/backend/core/permission.py +++ b/backend/core/permission.py @@ -14,7 +14,7 @@ from movai_core_shared.logger import Log from dal.models.scopestree import scopes -from dal.new_models import Role +from dal.models.role import Role LOGGER = Log.get_logger(__name__) diff --git a/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py b/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py index 52b5cac9..a1e2e307 100644 --- a/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py +++ b/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py @@ -30,6 +30,7 @@ from dal.models.var import Var from dal.new_models.configuration import Configuration +#from dal.scopes.configuration import Configuration from dal.scopes.fleetrobot import FleetRobot from dal.scopes.robot import Robot diff --git a/backend/endpoints/api/v1/frontend/ide/datavalidation.py b/backend/endpoints/api/v1/frontend/ide/datavalidation.py index fca6ff05..6c57b5a5 100644 --- a/backend/endpoints/api/v1/frontend/ide/datavalidation.py +++ b/backend/endpoints/api/v1/frontend/ide/datavalidation.py @@ -7,8 +7,8 @@ from movai_core_shared.logger import Log -from dal.models.scopestree import ScopesTree from dal.new_models.configuration import Configuration +#from dal.scopes.configuration import Configuration LOGGER = Log.get_logger(__name__) diff --git a/backend/endpoints/api/v1/frontend/ide/viewer.py b/backend/endpoints/api/v1/frontend/ide/viewer.py index 9525f3dc..9adb1266 100644 --- a/backend/endpoints/api/v1/frontend/ide/viewer.py +++ b/backend/endpoints/api/v1/frontend/ide/viewer.py @@ -21,7 +21,7 @@ LOGGER = Log.get_logger(__name__) try: - from movai_core_enterprise.new_models.annotation import Annotation + from movai_core_enterprise.models.annotation import Annotation from movai_core_enterprise.models.graphicscene import GraphicScene as GraphicSceneModel from movai_core_enterprise.scopes.graphicscene import GraphicScene as GraphicSceneScope except ImportError: diff --git a/backend/endpoints/api/v1/restapi.py b/backend/endpoints/api/v1/restapi.py index 4fe8d964..caf4312a 100644 --- a/backend/endpoints/api/v1/restapi.py +++ b/backend/endpoints/api/v1/restapi.py @@ -33,16 +33,17 @@ from dal.helpers.helpers import Helpers from dal.models.acl import NewACLManager from dal.models.lock import Lock +from dal.models.role import Role from dal.models.var import Var from dal.movaidb import MovaiDB -from dal.new_models import Application -from dal.new_models import Callback -from dal.new_models import Configuration -from dal.new_models import Node -from dal.new_models import Flow -from dal.new_models import Message -from dal.new_models import Ports -from dal.new_models.role import Role +from dal.new_models.application import Application +from dal.new_models.callback import Callback +from dal.new_models.configuration import Configuration +from dal.new_models.node import Node +from dal.new_models.flow import Flow +from dal.new_models.message import Message +from dal.new_models.ports import Ports + from dal.scopes.form import Form from dal.scopes.package import Package from dal.scopes.robot import Robot @@ -51,7 +52,7 @@ try: from movai_core_enterprise.message_client_handlers.metrics import Metrics - from movai_core_enterprise.new_models import Annotation + from movai_core_enterprise.models.annotation import Annotation from movai_core_enterprise.scopes.graphicscene import GraphicScene from movai_core_enterprise.scopes.layout import Layout from movai_core_enterprise.scopes.shareddatatemplate import SharedDataTemplate diff --git a/backend/endpoints/api/v2/base.py b/backend/endpoints/api/v2/base.py index 4e63af71..7397684a 100644 --- a/backend/endpoints/api/v2/base.py +++ b/backend/endpoints/api/v2/base.py @@ -26,14 +26,14 @@ from dal.movaidb import MovaiDB from dal.models.aclobject import AclObject -from dal.models.ldapconfig import LdapConfig -import dal.new_models.role from dal.models.internaluser import InternalUser +from dal.models.ldapconfig import LdapConfig +from dal.models.role import Role from dal.models.remoteuser import RemoteUser + from gd_node.protocols.http.middleware import redirect_not_found -from backend.http import IWebApp -Role = getattr(dal.new_models.role, "Role") +from backend.http import IWebApp class RestBaseClass: diff --git a/backend/endpoints/api/v2/role.py b/backend/endpoints/api/v2/role.py index 31b08673..12014360 100644 --- a/backend/endpoints/api/v2/role.py +++ b/backend/endpoints/api/v2/role.py @@ -8,18 +8,16 @@ Module that implements version 2 of the REST APi module/plugin """ - from abc import ABC, abstractmethod from typing import List + from aiohttp import web from aiohttp.web_response import Response -from backend.http import WebAppManager -from backend.endpoints.api.v2.base import BaseWebApp, RestBaseClass +from dal.models.role import Role -def Role(): - import dal.new_models.role - return dal.new_models.role.Role +from backend.http import WebAppManager +from backend.endpoints.api.v2.base import BaseWebApp, RestBaseClass class RoleRestBaseClass(RestBaseClass, ABC): @@ -32,7 +30,8 @@ def __init__(self) -> None: def extract_scope(self) -> None: """sets the scope the call is directed too.""" - self._scope = self.scope_classes.get("Role") + self._scope_name = Role.__name__ + self._scope = self.scope_classes.get(self._scope_name) @abstractmethod async def execute_imp(self) -> None: @@ -101,7 +100,7 @@ async def execute_imp(self) -> None: self.check_permissions() payload = await self._request.json() data = payload.get("data") - role_obj = Role().create(data["Label"], data["Resources"]) + role_obj = Role.create(data["Label"], data["Resources"]) role_obj.update_time() self._result["success"] = True self._result["name"] = data["Label"] @@ -139,7 +138,7 @@ async def execute_imp(self) -> None: """This method deletes an Role object from the DB.""" self.extract_object() self.check_permissions() - Role().remove(self._object_name) + Role.remove(self._object_name) self._result["success"] = True self._result["name"] = self._object_name diff --git a/backend/tools/new_user.py b/backend/tools/new_user.py index defb86a8..fce5fad3 100644 --- a/backend/tools/new_user.py +++ b/backend/tools/new_user.py @@ -10,7 +10,7 @@ from dal.models.acl import NewACLManager from dal.models.internaluser import InternalUser -from dal.new_models import Role +from dal.models.role import Role LOGGER = Log.get_logger(__name__) diff --git a/backend/tools/user_tool.py b/backend/tools/user_tool.py index c8faf227..c9037412 100755 --- a/backend/tools/user_tool.py +++ b/backend/tools/user_tool.py @@ -37,7 +37,7 @@ from movai_core_shared.logger import Log from dal.models.user import User -from dal.new_models import Role +from dal.models.role import Role from dal.models.internaluser import InternalUser CONVERT_COMMAND = "convert" From 7d087e3595f804ddcb6fd055dae27a646c981e7e Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Thu, 4 Jan 2024 22:19:43 +0200 Subject: [PATCH 25/26] fix sonarcloud issues --- backend/endpoints/api/v1/frontend/fleetmanager/statistics.py | 1 - backend/endpoints/api/v1/frontend/ide/datavalidation.py | 1 - 2 files changed, 2 deletions(-) diff --git a/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py b/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py index a1e2e307..52b5cac9 100644 --- a/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py +++ b/backend/endpoints/api/v1/frontend/fleetmanager/statistics.py @@ -30,7 +30,6 @@ from dal.models.var import Var from dal.new_models.configuration import Configuration -#from dal.scopes.configuration import Configuration from dal.scopes.fleetrobot import FleetRobot from dal.scopes.robot import Robot diff --git a/backend/endpoints/api/v1/frontend/ide/datavalidation.py b/backend/endpoints/api/v1/frontend/ide/datavalidation.py index 6c57b5a5..426c7335 100644 --- a/backend/endpoints/api/v1/frontend/ide/datavalidation.py +++ b/backend/endpoints/api/v1/frontend/ide/datavalidation.py @@ -8,7 +8,6 @@ from movai_core_shared.logger import Log from dal.new_models.configuration import Configuration -#from dal.scopes.configuration import Configuration LOGGER = Log.get_logger(__name__) From d639e3d78a0f42e0ce3d15d20e0b2358649a40a1 Mon Sep 17 00:00:00 2001 From: Erez Zomer Date: Thu, 4 Jan 2024 22:20:46 +0200 Subject: [PATCH 26/26] update setup.py to latest package of shared, dal and gd-node --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 03fb9aed..7ff498b1 100644 --- a/setup.py +++ b/setup.py @@ -13,9 +13,9 @@ "requests==2.28.2", "email-validator==2.0.0", "pytz==2022.7.1", - "movai-core-shared==2.5.0.9", - "data-access-layer==2.5.0.6", - "gd-node==2.5.0.5", + "movai-core-shared==2.5.0.10", + "data-access-layer==2.5.0.7", + "gd-node==2.5.0.7", ]