several docuntation etc

nfdi4plants · Apr 19, 2023 · f5e2bf4 · f5e2bf4
1 parent 2e1e4ee
commit f5e2bf4
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 11 deletions.
diff --git a/app/api/endpoints/webhooks.py b/app/api/endpoints/webhooks.py
@@ -57,11 +57,22 @@ async def ontology(request: Request, payload: PushWebhookPayload):
     branch = payload.after
     hash_id = payload.after
     # commits = payload.commits.pop()
-    commits = payload.commits[-1]
 
-    modified = commits.modified
-    added = commits.added
-    removed = commits.removed
+    #TODO: remove following ?
+    #commits = payload.commits[-1]
+
+    #modified = commits.modified
+    #added = commits.added
+    #removed = commits.removed
+
+    # TODO: check if thats working
+    commits = payload.commits
+    for commit in commits:
+       print("current commit", commit)
+       modified = modified + commit.modified
+       added = added + commit.added
+       removed = removed + commit.removed
+
 
     update_files = modified + added
 

diff --git a/app/neo4j_conc/neo4jConnection.py b/app/neo4j_conc/neo4jConnection.py
@@ -30,6 +30,11 @@ def write(self, batch_queries):
 
 
     def read(self, query):
+        """
+        Reads some data from neo4j database
+        :param query: neo4j query
+        :return: neo4j record as list?
+        """
         with self.lock:
             with self.driver.session() as session:
                 # async with await session.begin_transaction() as tx:

diff --git a/app/neo4j_conc/term_operations.py → app/neo4j_conc/ontology_operations.py b/app/neo4j_conc/term_operations.py → app/neo4j_conc/ontology_operations.py
@@ -49,17 +49,37 @@ def connect_term_relationships_apoc():
     return query
 
 
-def update_ontologies():
+def add_ontologies(data, batch_size: int = 40000) -> list:
+    batch_queries = []
+
     query = '''
             UNWIND $rows AS row
             MERGE (o:Ontology {name: row.name})
-            SET o.lastUpdated = row.lastUpdated
-            SET o.author = row.author
-            SET o.version = row.version
+            SET o.lastUpdated = COALESCE(o.lastUpdated,row.lastUpdated)
+            SET o.author = COALESCE(o.author,row.author)
+            SET o.version = COALESCE(o.version,row.version)
             SET o.generated = COALESCE(o.generated,row.generated)
             SET o.importedFrom = COALESCE(o.importedFrom,row.importedFrom)
             RETURN count(*) as total
             '''
+
+    for i in range(0, len(data), batch_size):
+        batch_data = data.iloc[i:i + batch_size]
+        batch_queries = []
+        # batch_data = []
+        print("query", batch_queries)
+        for index, row in batch_data.iterrows():
+            params = {"lastUpdated": row["Id"],
+                      "version": row["Name"],
+                      "description": row["Description"],
+                      "version": row["Version"],
+                      "generated": str(row["Authors"]),
+                      "importedFrom": str(row["TemplateJson"]),
+                      "organisation": row["Organisation"],
+                      }
+            batch_queries.append((query, params))
+
+
     return query
 
 

diff --git a/app/neo4j_conc/template_operations.py b/app/neo4j_conc/template_operations.py
@@ -8,8 +8,13 @@
 from app.helpers.models.templates.template import Template
 
 
-def add_template(data, batch_size=10000):
-
+def add_template(data, batch_size: int = 10000) -> list:
+    """
+    Build a query to add one or more templates to the neo4j database
+    :param data: pandas dataframe
+    :param batch_size: size of batches
+    :return: list of queries
+    """
     batch_queries = []
 
     print("data", data)
@@ -72,7 +77,11 @@ def delete_template(template_id):
 
     return batch_queries
 
-def delete_all_templates():
+def delete_all_templates() -> list:
+    """
+    Build a query to delete all templates in neo4j database
+    :return: list of queries
+    """
     batch_queries = []
     query = '''
             MATCH (t:Template) DELETE t