diff --git a/examples/notebooks/use-cases/LocalValidation.ipynb b/examples/notebooks/use-cases/LocalValidation.ipynb new file mode 100644 index 00000000..092a3687 --- /dev/null +++ b/examples/notebooks/use-cases/LocalValidation.ipynb @@ -0,0 +1,425 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Resolving Strategies\n", + "\n", + "* Example on how to use resolving strategies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2019-09-23T18:50:20.068658Z", + "start_time": "2019-09-23T18:50:19.054054Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "from kgforge.core import KnowledgeGraphForge\n", + "from kgforge.specializations.resources import Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# import getpass\n", + "# TOKEN = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", + "BUCKET = \"dke/kgforge\"\n", + "forge = KnowledgeGraphForge(\"../use-cases/prod-forge-nexus.yml\",\n", + " endpoint=endpoint, \n", + " bucket=BUCKET,\n", + "# token=TOKEN,\n", + " debug=True\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "some_str = \"\"\"\n", + "{\n", + " \"type\": [\n", + " \"Dataset\",\n", + " \"Trace\",\n", + " \"Entity\",\n", + " \"SingleCellSimulationTrace\"\n", + " ],\n", + " \"id\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\",\n", + " \"annotation\": [\n", + " {\n", + " \"type\": [\n", + " \"Annotation\",\n", + " \"MTypeAnnotation\"\n", + " ],\n", + " \"hasBody\": {\n", + " \"id\": \"ilx:0738236\",\n", + " \"type\": [\n", + " \"AnnotationBody\",\n", + " \"MType\"\n", + " ],\n", + " \"label\": \"VPL_TC\"\n", + " },\n", + " \"name\": \"M-type Annotation\"\n", + " },\n", + " {\n", + " \"type\": [\n", + " \"Annotation\",\n", + " \"ETypeAnnotation\"\n", + " ],\n", + " \"hasBody\": {\n", + " \"id\": \"ilx:0738254\",\n", + " \"type\": [\n", + " \"AnnotationBody\",\n", + " \"EType\"\n", + " ],\n", + " \"label\": \"cNAD_ltb\"\n", + " },\n", + " \"name\": \"E-type Annotation\"\n", + " }\n", + " ],\n", + " \"identifier\": \"dNAD_ltb\",\n", + " \"note\": \"Simulated Thalamus cell\",\n", + " \"dateCreated\": {\n", + " \"@value\": \"2021-12-08T18:03:31.001661+01:00\",\n", + " \"type\": \"xsd:dateTime\"\n", + " },\n", + " \"brainLocation\": {\n", + " \"type\": \"BrainLocation\",\n", + " \"brainRegion\": {\n", + " \"id\": \"http://api.brain-map.org/api/v2/data/Structure/718\",\n", + " \"label\": \"Ventral posterolateral nucleus of the thalamus\"\n", + " }\n", + " },\n", + " \"contribution\": [\n", + " {\n", + " \"type\": \"Contribution\",\n", + " \"agent\": {\n", + " \"id\": \"https://bbp.epfl.ch/nexus/v1/realms/bbp/users/tuncel\",\n", + " \"type\": \"Agent\",\n", + " \"label\": \"Anil Tuncel\"\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"Contribution\",\n", + " \"agent\": {\n", + " \"id\": \"https://bbp.epfl.ch/nexus/v1/realms/bbp/users/iavarone\",\n", + " \"type\": \"Agent\",\n", + " \"label\": \"Elisabetta Iavarone\"\n", + " }\n", + " }\n", + " ],\n", + " \"description\": \"This dataset is about simulated electrophysiology traces for cell instance dNAD_ltb. The dataset contains one distribution of the traces in NWB file format.\",\n", + " \"subject\": {\n", + " \"type\": \"Subject\",\n", + " \"species\": {\n", + " \"id\": \"ncbitaxon:10090\",\n", + " \"label\": \"Mus musculus\"\n", + " }\n", + " },\n", + " \"distribution\": {\n", + " \"type\": \"DataDownload\",\n", + " \"contentSize\": {\n", + " \"unitCode\": \"bytes\",\n", + " \"value\": 2347304\n", + " },\n", + " \"digest\": {\n", + " \"algorithm\": \"SHA-256\",\n", + " \"value\": \"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\"\n", + " },\n", + " \"encodingFormat\": \"application/nwb\",\n", + " \"name\": \"dNAD_ltb.nwb\",\n", + " \"contentUrl\": \"https://bbp.epfl.ch/nexus/v1/files/bbp/uniprot/63c13f11-4dde-42fa-81cd-74fe6a68f35c\",\n", + " \"atLocation\": {\n", + " \"type\": \"Location\",\n", + " \"store\": {\n", + " \"id\": \"https://bbp.epfl.ch/data/bbp/uniprot/dc51e00c-584f-4f8b-a5da-df85d2416d79\",\n", + " \"type\": \"RemoteDiskStorage\",\n", + " \"_rev\": 1\n", + " },\n", + " \"location\": \"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\"\n", + " }\n", + " },\n", + " \"objectOfStudy\": {\n", + " \"type\": \"ObjectOfStudy\",\n", + " \"id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", + " \"label\": \"Single Cell\"\n", + " }\n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "resource = forge.from_json(json.loads(some_str))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use a debug mode in pyschacl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _validate_one\n", + " True\n" + ] + } + ], + "source": [ + "forge.validate(resource, type_=\"Entity\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _register_one\n", + " False\n", + " RegistrationError: Resource 'https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210' failed to validate against the constraints defined in schema 'https://neuroshapes.org/dash/dataset'. Reason: Value does not have all the shapes in the sh:and enumeration for the shape: https://neuroshapes.org/commons/minds/shapes/MINDSShape\n" + ] + } + ], + "source": [ + "forge.register(resource, schema_id=\"datashapes:dataset\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "jonis = \"\"\"\n", + "{\n", + " \"atlasRelease\": {\n", + " \"id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/197d151d-4ed2-4549-85fd-6c51bd471262\",\n", + " \"type\": [\"AtlasRelease\", \"BrainAtlasRelease\"]\n", + " },\n", + " \"brainLocation\": {\n", + " \"brainRegion\": {\n", + " \"id\": \"http://api.brain-map.org/api/v2/data/Structure/549\",\n", + " \"label\": \"Thalamus\"\n", + " },\n", + " \"type\": \"BrainLocation\"\n", + " },\n", + " \"circuitConfigPath\": {\n", + " \"type\": \"DataDownload\",\n", + " \"url\": \"file:///will/this/path/exist\"\n", + " },\n", + " \"circuitType\": \"Test registration\",\n", + " \"description\": \"Test registration, to be deprecated\",\n", + " \"name\": \"Test Circuit registration\",\n", + " \"type\": \"DetailedCircuit\",\n", + " \"wasGeneratedBy\": [{\"id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/9be40e75-8744-415b-b0b4-e4074ff54a8f\"},\n", + " {\"id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/2b5819a1-c5e9-42c4-8da7-864d5e1e0a7e\"}],\n", + " \"nodeCollection\" : {\n", + " \"type\": \"NodeCollection\",\n", + " \"name\": \"My node collection\",\n", + " \"circuitCellProperties\": {\n", + " \"type\" : \"CircuitCellProperties\",\n", + " \"name\": \"My cell properties\"\n", + " },\n", + " \"memodelRelease\": {\n", + " \"type\" : \"MEModelRelease\",\n", + " \"name\": \"Some model release\"\n", + " }\n", + " }\n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "joni_resource = forge.from_json(json.loads(jonis))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"atlasRelease\": {\n", + " \"@id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/197d151d-4ed2-4549-85fd-6c51bd471262\",\n", + " \"@type\": [\n", + " \"AtlasRelease\",\n", + " \"BrainAtlasRelease\"\n", + " ]\n", + " },\n", + " \"brainLocation\": {\n", + " \"brainRegion\": {\n", + " \"@id\": \"http://api.brain-map.org/api/v2/data/Structure/549\",\n", + " \"label\": \"Thalamus\"\n", + " },\n", + " \"@type\": \"BrainLocation\"\n", + " },\n", + " \"circuitConfigPath\": {\n", + " \"@type\": \"DataDownload\",\n", + " \"url\": \"file:///will/this/path/exist\"\n", + " },\n", + " \"circuitType\": \"Test registration\",\n", + " \"description\": \"Test registration, to be deprecated\",\n", + " \"name\": \"Test Circuit registration\",\n", + " \"@type\": \"DetailedCircuit\",\n", + " \"nodeCollection\": {\n", + " \"@type\": \"NodeCollection\",\n", + " \"name\": \"My node collection\",\n", + " \"circuitCellProperties\": {\n", + " \"@type\": \"CircuitCellProperties\",\n", + " \"name\": \"My cell properties\"\n", + " },\n", + " \"memodelRelease\": {\n", + " \"@type\": \"MEModelRelease\",\n", + " \"name\": \"Some model release\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(forge.as_jsonld(joni_resource), indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _register_one\n", + " False\n", + " RegistrationError: Schema 'https://neuroshapes.org/DetailedCircuit' could not be resolved in 'dke/kgforge'\n" + ] + } + ], + "source": [ + "forge.register(joni_resource, schema_id=\"https://neuroshapes.org/DetailedCircuit\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Test Circuit registration'" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "joni_resource.name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _validate_one\n", + " True\n" + ] + } + ], + "source": [ + "forge.validate(joni_resource, type_='DetailedCircuit')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.13 ('kgforge')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "9ac393a5ddd595f2c78ea58b15bf8d269850a4413729cbea5c5fae9013762763" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/kgforge/core/archetypes/model.py b/kgforge/core/archetypes/model.py index 8bc36b53..579b709f 100644 --- a/kgforge/core/archetypes/model.py +++ b/kgforge/core/archetypes/model.py @@ -150,18 +150,18 @@ def schema_id(self, type: str) -> str: not_supported() def validate(self, data: Union[Resource, List[Resource]], - execute_actions_before: bool, type_: str) -> None: + execute_actions_before: bool, type_: str, debug: bool=False) -> None: # Replace None by self._validate_many to switch to optimized bulk validation. run(self._validate_one, None, data, execute_actions=execute_actions_before, - exception=ValidationError, monitored_status="_validated", type_=type_) + exception=ValidationError, monitored_status="_validated", type_=type_, debug=debug) - def _validate_many(self, resources: List[Resource], type_: str) -> None: + def _validate_many(self, resources: List[Resource], type_: str, debug: bool) -> None: # Bulk validation could be optimized by overriding this method in the specialization. # POLICY Should reproduce self._validate_one() and execution._run_one() behaviours. not_supported() @abstractmethod - def _validate_one(self, resource: Resource, type_: str) -> None: + def _validate_one(self, resource: Resource, type_: str, debug: bool) -> None: # POLICY Should notify of failures with exception ValidationError including a message. pass diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index 0af43410..eefea853 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -154,7 +154,8 @@ def mapper(self) -> Optional[Callable]: # [C]RUD. def register( - self, data: Union[Resource, List[Resource]], schema_id: str = None + self, data: Union[Resource, List[Resource]], schema_id: str = None, + debug: bool = False ) -> None: # Replace None by self._register_many to switch to optimized bulk registration. run( @@ -163,6 +164,7 @@ def register( data, required_synchronized=False, execute_actions=True, + catch_exceptions=not debug, exception=RegistrationError, monitored_status="_synchronized", schema_id=schema_id, @@ -316,7 +318,7 @@ def _download_one( # CR[U]D. def update( - self, data: Union[Resource, List[Resource]], schema_id: Optional[str] + self, data: Union[Resource, List[Resource]], schema_id: Optional[str], debug: bool = False ) -> None: # Replace None by self._update_many to switch to optimized bulk update. run( @@ -329,6 +331,7 @@ def update( exception=UpdatingError, monitored_status="_synchronized", schema_id=schema_id, + catch_exceptions=not debug ) def _update_many(self, resources: List[Resource], schema_id: Optional[str]) -> None: @@ -343,7 +346,7 @@ def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: # TODO This operation might be abstracted here when other stores will be implemented. pass - def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + def tag(self, data: Union[Resource, List[Resource]], value: str, debug: bool = True) -> None: # Replace None by self._tag_many to switch to optimized bulk tagging. # POLICY If tagging modify the resource, run() should have status='_synchronized'. run( @@ -352,6 +355,7 @@ def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: data, id_required=True, required_synchronized=True, + catch_exceptions=not debug, exception=TaggingError, value=value, ) @@ -369,7 +373,7 @@ def _tag_one(self, resource: Resource, value: str) -> None: # CRU[D]. - def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + def deprecate(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: # Replace None by self._deprecate_many to switch to optimized bulk deprecation. run( self._deprecate_one, @@ -377,6 +381,7 @@ def deprecate(self, data: Union[Resource, List[Resource]]) -> None: data, id_required=True, required_synchronized=True, + catch_exceptions=not debug, exception=DeprecationError, monitored_status="_synchronized", ) @@ -461,7 +466,7 @@ def _elastic(self, query: str) -> List[Resource]: # Versioning. - def freeze(self, data: Union[Resource, List[Resource]]) -> None: + def freeze(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: # Replace None by self._freeze_many to switch to optimized bulk freezing. run( self._freeze_one, @@ -469,6 +474,7 @@ def freeze(self, data: Union[Resource, List[Resource]]) -> None: data, id_required=True, required_synchronized=True, + catch_exceptions=not debug, exception=FreezingError, ) diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 0c57eea0..bd09a61f 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -31,6 +31,8 @@ from kgforge.core.commons.dictionaries import with_defaults from kgforge.core.commons.exceptions import ResolvingError from kgforge.core.commons.execution import catch +from kgforge.core.commons.actions import (collect_lazy_actions, + execute_lazy_actions) from kgforge.core.commons.imports import import_class from kgforge.core.commons.strategies import ResolvingStrategy from kgforge.core.commons.formatter import Formatter @@ -303,7 +305,7 @@ def template( def validate( self, data: Union[Resource, List[Resource]], - execute_actions_before: bool=False, + execute_actions_before: bool = False, type_: str=None ) -> None: """ @@ -317,7 +319,8 @@ def validate( :param type_: the type to validate the data against it. If None, the validation function will look for a type attribute in the Resource :return: None """ - self._model.validate(data, execute_actions_before, type_=type_) + debug = self._debug + self._model.validate(data, execute_actions_before, type_, debug) # Resolving User Interface. @@ -669,6 +672,19 @@ def elastic( :return: List[Resource] """ return self._store.elastic(query, debug, limit, offset) + + @catch + @staticmethod + def execute_lazy_actions(resources: Union[List[Resource], Resource]): + """ + Execute any lazy action present in a given resource or a list of resources + + :param resources: The resources or list of resources from which actions will be executed + """ + resources = [resources] if not isinstance(resources, List) else resources + for resource in resources: + lazy_actions = collect_lazy_actions(resource) + execute_lazy_actions(resource, lazy_actions) @catch def download( @@ -705,8 +721,8 @@ def register( :param data: the resources to register :param schema_id: an identifier of the schema the registered resources should conform to """ - #self._store.mapper = self._store.mapper(self) - self._store.register(data, schema_id) + + self._store.register(data, schema_id, debug=self._debug) # No @catch because the error handling is done by execution.run(). def update( @@ -718,7 +734,7 @@ def update( :param data: the resources to update :param schema_id: an identifier of the schema the updated resources should conform to """ - self._store.update(data, schema_id) + self._store.update(data, schema_id, debug=self._debug) # No @catch because the error handling is done by execution.run(). def deprecate(self, data: Union[Resource, List[Resource]]) -> None: @@ -727,7 +743,7 @@ def deprecate(self, data: Union[Resource, List[Resource]]) -> None: :param: the resources to deprecate """ - self._store.deprecate(data) + self._store.deprecate(data, debug=self._debug) # Versioning User Interface. @@ -739,7 +755,7 @@ def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: :param data: the resources to tag :param value: the tag value """ - self._store.tag(data, value) + self._store.tag(data, value, debug=self._debug) # No @catch because the error handling is done by execution.run(). def freeze(self, data: Union[Resource, List[Resource]]) -> None: @@ -749,7 +765,7 @@ def freeze(self, data: Union[Resource, List[Resource]]) -> None: :param data: the resources to freeze """ - self._store.freeze(data) + self._store.freeze(data, debug=self._debug) # Files Handling User Interface. diff --git a/kgforge/specializations/models/demo_model.py b/kgforge/specializations/models/demo_model.py index 32815986..81d807b1 100644 --- a/kgforge/specializations/models/demo_model.py +++ b/kgforge/specializations/models/demo_model.py @@ -81,7 +81,7 @@ def mapping(self, entity: str, source: str, type: Callable) -> Mapping: # Validation. - def _validate_one(self, resource: Resource, type_: str) -> None: + def _validate_one(self, resource: Resource, type_: str, debug: bool) -> None: """ Validates the model against a given type provided by type_ parameter. If type_ is None then it looks for type attribute in resource. diff --git a/kgforge/specializations/models/rdf/directory_service.py b/kgforge/specializations/models/rdf/directory_service.py index 46bf4598..de4cd1d6 100644 --- a/kgforge/specializations/models/rdf/directory_service.py +++ b/kgforge/specializations/models/rdf/directory_service.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Dict, Tuple -from pyshacl import Validator +from pyshacl import validate from rdflib import Graph, URIRef from rdflib.util import guess_format @@ -43,9 +43,9 @@ def materialize(self, iri: URIRef) -> NodeProperties: attrs["properties"] = props return NodeProperties(**attrs) - def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: - validator = Validator(data_graph, shacl_graph=self._graph) - return validator.run() + def _validate(self, iri: str, data_graph: Graph, debug: bool) -> Tuple[bool, Graph, str]: + result = validate(data_graph, shacl_graph=self._graph, debug=debug) + return result def resolve_context(self, iri: str) -> Dict: if iri in self._context_cache: diff --git a/kgforge/specializations/models/rdf/service.py b/kgforge/specializations/models/rdf/service.py index 6f581968..c1a4dbf8 100644 --- a/kgforge/specializations/models/rdf/service.py +++ b/kgforge/specializations/models/rdf/service.py @@ -155,7 +155,7 @@ def materialize(self, iri: URIRef) -> NodeProperties: """ raise NotImplementedError() - def validate(self, resource: Resource, type_: str): + def validate(self, resource: Resource, type_: str, debug: bool): try: if isinstance(resource.type, list) and type_ is None: raise ValueError("Resource has list of types as attribute and type_ parameter is not specified. " @@ -168,10 +168,10 @@ def validate(self, resource: Resource, type_: str): raise TypeError("resource requires a type attribute") else: data_graph = as_graph(resource, False, self.context, None, None) - return self._validate(shape_iri, data_graph) + return self._validate(shape_iri, data_graph, debug) @abstractmethod - def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: + def _validate(self, iri: str, data_graph: Graph, debug: bool) -> Tuple[bool, Graph, str]: raise NotImplementedError() @abstractmethod diff --git a/kgforge/specializations/models/rdf/store_service.py b/kgforge/specializations/models/rdf/store_service.py index 621f8fea..a46a02bc 100644 --- a/kgforge/specializations/models/rdf/store_service.py +++ b/kgforge/specializations/models/rdf/store_service.py @@ -15,7 +15,7 @@ import json -from pyshacl import Validator +from pyshacl import validate from kgforge.core import Resource from kgforge.core.commons.exceptions import RetrievalError @@ -57,11 +57,11 @@ def materialize(self, iri: URIRef) -> NodeProperties: attrs["properties"] = props return NodeProperties(**attrs) - def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: + def _validate(self, iri: str, data_graph: Graph, debug: bool) -> Tuple[bool, Graph, str]: # _type_shape will make sure all the shapes for this type are in the graph self._type_shape(iri) - validator = Validator(data_graph, shacl_graph=self._graph) - return validator.run() + result = validate(data_graph, shacl_graph=self._graph, debug=debug) + return result def resolve_context(self, iri: str) -> Dict: if iri in self._context_cache: diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index d6edb49f..cfca7620 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -107,13 +107,15 @@ def schema_id(self, type: str) -> str: except KeyError: raise ValueError("type not found") - def validate(self, data: Union[Resource, List[Resource]], execute_actions_before: bool, type_: str) -> None: + def validate(self, data: Union[Resource, List[Resource]], execute_actions_before: bool, type_: str, + debug: bool = False) -> None: run(self._validate_one, self._validate_many, data, execute_actions=execute_actions_before, - exception=ValidationError, monitored_status="_validated", type_=type_) + exception=ValidationError, monitored_status="_validated", catch_exceptions=not debug, + type_=type_, debug=debug) - def _validate_many(self, resources: List[Resource], type_: str) -> None: + def _validate_many(self, resources: List[Resource], type_: str, debug: bool) -> None: for resource in resources: - conforms, graph, _ = self.service.validate(resource, type_=type_) + conforms, graph, _ = self.service.validate(resource, type_=type_, debug=debug) if conforms: resource._validated = True action = Action(self._validate_many.__name__, conforms, None) @@ -125,8 +127,8 @@ def _validate_many(self, resources: List[Resource], type_: str) -> None: action = Action(self._validate_many.__name__, conforms, ValidationError(message)) resource._last_action = action - def _validate_one(self, resource: Resource, type_: str) -> None: - conforms, _, report = self.service.validate(resource, type_) + def _validate_one(self, resource: Resource, type_: str, debug: bool) -> None: + conforms, _, report = self.service.validate(resource, type_=type_, debug=debug) if conforms is False: raise ValidationError("\n" + report) diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index f02727d8..75e9292f 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -39,7 +39,7 @@ from rdflib import Graph from rdflib.plugins.sparql.parser import Query from datetime import datetime -from requests import HTTPError +from requests import HTTPError, Response from kgforge.core import Resource from kgforge.core.archetypes import Store @@ -139,7 +139,8 @@ def mapper(self) -> Optional[DictionaryMapper]: return DictionaryMapper def register( - self, data: Union[Resource, List[Resource]], schema_id: str = None + self, data: Union[Resource, List[Resource]], schema_id: str = None, + debug: bool = False ) -> None: run( self._register_one, @@ -148,6 +149,7 @@ def register( required_synchronized=False, execute_actions=True, exception=RegistrationError, + catch_exceptions=not debug, monitored_status="_synchronized", schema_id=schema_id, ) @@ -588,13 +590,14 @@ def _update_one(self, resource: Resource, schema_id: str) -> None: else: self.service.sync_metadata(resource, response.json()) - def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + def tag(self, data: Union[Resource, List[Resource]], value: str, debug: bool = False) -> None: run( self._tag_one, self._tag_many, data, id_required=True, required_synchronized=True, + catch_exceptions=not debug, exception=TaggingError, value=value, ) @@ -638,13 +641,14 @@ def _tag_one(self, resource: Resource, value: str) -> None: # CRU[D]. - def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + def deprecate(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: run( self._deprecate_one, self._deprecate_many, data, id_required=True, required_synchronized=True, + catch_exceptions=not debug, exception=DeprecationError, monitored_status="_synchronized", ) @@ -1071,21 +1075,13 @@ def rewrite_uri(self, uri: str, context: Context, **kwargs) -> str: return uri -def _error_message(error: HTTPError) -> str: - def format_message(msg): - return "".join([msg[0].lower(), msg[1:-1], msg[-1] if msg[-1] != "." else ""]) +def format_message(msg): + return "".join([msg[0].lower(), msg[1:-1], msg[-1] if msg[-1] != "." else ""]) +def _error_message(error: HTTPError) -> str: try: - error_json = error.response.json() - messages = [] - reason = error_json.get("reason", None) - details = error_json.get("details", None) - if reason: - messages.append(format_message(reason)) - if details: - messages.append(format_message(details)) - messages = messages if reason or details else [str(error)] - return ". ".join(messages) + error_response = error.response + return _error_from_response(error_response) except Exception as e: pass try: @@ -1093,6 +1089,34 @@ def format_message(msg): except Exception: return format_message(str(error)) +def _error_from_response(response: Response) -> str: + def details_string(details: dict) -> str: + string = f"\nReason: {detail.pop('resultMessage')} "\ + f"for the shape: {detail.pop('sourceShape')}.\nError details:\n" + for key, value in details.items(): + string += f"{key}:\t{value}\n" + return string + messages = [] + error_json = response.json() + reason = error_json.get("reason", None) + details = error_json.get("details", None) + if reason: + messages.append(reason) + if details: + result = details.get('result', None) + if result: + the_details = result.get("detail", None) + if the_details: + if isinstance(the_details, list): + for detail in the_details: + messages.append(details_string(detail)) + elif isinstance(the_details, dict): + messages.append(details_string(detail)) + else: + messages.append(str(the_details)) + else: + messages.append(str(result)) + return ". ".join(messages) def _create_select_query(vars_, statements, distinct, search_in_graph): where_clauses = ( diff --git a/tests/specializations/models/test_demo_model.py b/tests/specializations/models/test_demo_model.py index 43ebbc20..f53885e0 100644 --- a/tests/specializations/models/test_demo_model.py +++ b/tests/specializations/models/test_demo_model.py @@ -44,8 +44,8 @@ def validate(capsys, model, data, rc, err, msg): @when("I validate the resource. An exception is raised. The printed report mentions an error: 'Exception: exception raised'.") def validate_exception(monkeypatch, capsys, model, data): - def _validate_one(_, x, type_: str): raise Exception("exception raised") + def _validate_one(_, x, type_: str, debug: bool): raise Exception("exception raised") monkeypatch.setattr("kgforge.specializations.models.demo_model.DemoModel._validate_one", _validate_one) - model.validate(data, execute_actions_before=False, type_="Person") + model.validate(data, execute_actions_before=False, type_="Person", debug=False) out = capsys.readouterr().out[:-1] assert out == f" _validate_one\n False\n Exception: exception raised" diff --git a/tests/specializations/models/test_rdf_model.py b/tests/specializations/models/test_rdf_model.py index 4546a771..5fc7bee3 100644 --- a/tests/specializations/models/test_rdf_model.py +++ b/tests/specializations/models/test_rdf_model.py @@ -99,20 +99,20 @@ def test_type_to_schema(self, rdf_model: RdfModel, type_): assert rdf_model.schema_id(type_) == TYPES_SCHEMAS_MAP[type_] def test_validate_one(self, rdf_model: RdfModel, valid_activity_resource): - rdf_model.validate(valid_activity_resource, False, type_="Activity") + rdf_model.validate(valid_activity_resource, False, type_="Activity", debug=False) def test_validate_one_fail(self, rdf_model: RdfModel, invalid_activity_resource): with pytest.raises(ValidationError): - rdf_model._validate_one(invalid_activity_resource, type_="Activity") + rdf_model._validate_one(invalid_activity_resource, type_="Activity", debug=False) def test_validate_with_schema(self, rdf_model: RdfModel, valid_activity_resource): - rdf_model.validate(valid_activity_resource, False, type_="Activity") + rdf_model.validate(valid_activity_resource, False, type_="Activity", debug=False) def test_validate_many(self, rdf_model: RdfModel, valid_activity_resource, invalid_activity_resource): resources = [valid_activity_resource, invalid_activity_resource] - rdf_model.validate(resources, False, type_="Activity") + rdf_model.validate(resources, False, type_="Activity", debug=False) assert valid_activity_resource._validated is True assert invalid_activity_resource._validated is False assert (valid_activity_resource._last_action.operation ==