Adding pulling of individual synapses

src/vfb_connect/cross_server_tools.py

@@ -823,21 +823,18 @@ def get_terms_by_xref(self, xrefs: iter, db='', summary=True, return_dataframe=T
 
         return self.neo_query_wrapper.get_terms_by_xref(xrefs, db=db, summary=summary, return_dataframe=False)
 
-    def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, return_just_ids=True, verbose=False):
-        """Map a list external DB IDs to VFB IDs
+    def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, return_just_ids=True, verbose=False, datasource_only=True):
+        """Map a list of external DB IDs to VFB short_form IDs
 
-        :param acc: An iterable (e.g. a list) of external IDs (e.g. neuprint bodyIDs). Can be in the form of 'db:acc' or just 'acc'.
+        :param acc: An iterable (e.g. a list) of external DB IDs.
         :param db: optional specify the VFB id (short_form) of an external DB to map to. (use get_dbs to find options)
         :param id_type: optionally specify an external id_type
         :param reverse_return: Boolean: Optional (see return)
         :param return_just_ids: Boolean: Optional (see return)
-        :param verbose: Optional. If `True`, prints the running query and found terms. Default `False`.
         :return: if `reverse_return` is False:
-            dict { acc : [{ db: <db> : vfb_id : <VFB_id> }
+                dict { acc : [{ db: <db> : vfb_id : <VFB_id> }
             Return if `reverse_return` is `True`:
-            dict { VFB_id : [{ db: <db> : acc : <acc> }
-            if `return_just_ids` is `True`:
-            return just the VFB_ids in a list
+                dict { vfb_id : [{ db: <db> : acc : <acc> }
         """
         if isinstance(acc, str):
             if ':' in acc and db == '':
@@ -852,19 +849,27 @@ def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, retur
                         new_acc.append(temp_acc)
                     else:
                         new_acc.append(xref.split(':')[-1])
+                else:
+                    new_acc.append(xref)
             acc = new_acc
         if isinstance(acc, list) and all(isinstance(x, int) for x in acc):
             acc = [str(x) for x in acc]
             print(f"Converted to strings: {acc}") if verbose else None
         if db in VFB_DBS_2_SYMBOLS.keys():
             db = VFB_DBS_2_SYMBOLS[db]
-        result = self.neo_query_wrapper.xref_2_vfb_id(acc=acc, db=db, id_type=id_type, reverse_return=reverse_return, verbose=verbose)
+        if db not in self.get_dbs():
+            db = self.lookup_id(db)
+        result = self.neo_query_wrapper.xref_2_vfb_id(acc=acc, db=db, id_type=id_type, reverse_return=reverse_return, verbose=verbose, datasource_only=datasource_only)
         print(result) if verbose else None
         if return_just_ids & reverse_return:
             return [x.key for x in result]
         if return_just_ids and not reverse_return:
             id_list = []
             for id in acc:
+                if id not in result.keys():
+                    print(f"No match found for {id} returning xref {':'.join([db,id])}") if verbose else None
+                    id_list.append(":".join([db,id]))
+                    continue
                 id_list.append(result[id][0]['vfb_id']) # This takes the first match only
                 if len(result[id]) > 1:
                     print(f"Multiple matches found for {id}: {result[id]}")

src/vfb_connect/neo/query_wrapper.py

@@ -445,18 +445,16 @@ def vfb_id_2_neuprint_bodyID(self, vfb_id, db=''):
         mapping = self.vfb_id_2_xrefs(vfb_id, db=db, reverse_return=True)
         return [int(k) for k, v in mapping.items()]
 
-    def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbose=False):
-        """Map a list external DB IDs to VFB IDs
-
-          :param acc: An iterable (e.g. a list) of external IDs (e.g. neuprint bodyIDs).
-          :param db: optional specify the VFB id (short_form) of an external DB to map to. (use get_dbs to find options)
-          :param id_type: optionally specify an external id_type
-          :param reverse_return: Boolean: Optional (see return)
-          :return: if `reverse_return` is False:
-                dict { acc : [{ db: <db> : vfb_id : <VFB_id> }
-              Return if `reverse_return` is `True`:
-                dict { VFB_id : [{ db: <db> : acc : <acc> }
-          """
+    def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbose=False, datasource_only=False):
+        """Map a list of external DB IDs to VFB short_form IDs
+
+        :param acc: An iterable (e.g., a list) of external DB IDs.
+        :param db: Optional. Specify the VFB ID (short_form) of an external database to map to. (Use `get_dbs()` to find options).
+        :param id_type: Optional. Specify an external ID type to filter the mapping results.
+        :param reverse_return: Optional. If `True`, returns the results in reverse order. Default is `False`.
+        :return: A dictionary of mappings from external DB IDs to VFB IDs.
+        :rtype: dict
+        """
         if isinstance(acc, str):
             acc = [acc]
         match = "MATCH (s:Individual)<-[r:database_cross_reference]-(i:Entity) WHERE"
@@ -468,6 +466,8 @@ def xref_2_vfb_id(self, acc=None, db='', id_type='', reverse_return=False, verbo
             conditions.append("s.short_form = '%s'" % db)
         if id_type:
             conditions.append("r.id_type = '%s'" % id_type)
+        if datasource_only:
+            conditions.append("s.is_data_source = [True]")
         condition_clauses = ' AND '.join(conditions)
         ret = "RETURN r.accession[0] as key, " \
               "collect({ db: s.short_form, vfb_id: i.short_form }) as mapping"

src/vfb_connect/schema/test/vfb_term_test.py

@@ -502,5 +502,18 @@ def test_vfbterm_xref(self):
         print(self.vfb.xref_2_vfb_id(term.xref_id, return_just_ids=True, verbose=True))
         self.assertEqual(self.vfb.xref_2_vfb_id(term.xref_id, return_just_ids=True)[0], term.id)
 
+    def test_load_synapes(self):
+        term = self.vfb.term('VFB_jrchk6dr')
+        print("got term ", term)
+        term.load_skeleton(template='JRC2018Unisex')
+        con = term.load_skeleton_synaptic_connections(verbose=True).to_dict('records')
+        print(con[0])
+        self.assertGreater(len(con),10)
+        term = self.vfb.term('VFB_00102gjr')
+        print("got term ", term)
+        con = term.load_skeleton_synaptic_connections().to_dict('records')
+        print(con[0])
+        self.assertGreater(len(con),10)
+
 if __name__ == "__main__":
     unittest.main()

src/vfb_connect/schema/vfb_term.py

@@ -12,6 +12,9 @@
 
 import webbrowser
 
+NEUPRINT_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InZmYndvcmtzaG9wLm5ldXJvZmx5MjAyMEBnbWFpbC5jb20iLCJsZXZlbCI6Im5vYXV0aCIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg2Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tWXFDN21NRXd3TlEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQU1adXVjbU5zaXhXZDRhM0VyTTQ0ODBMa2IzNDdvUlpfUS9zOTYtYy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3OTQwOTE4ODd9.ceg4mrj2o-aOhK0NHNGmBacg8R34PBPoLBwhCo4uOCQ'
+CATMAID_TOKEN = None
+
 neuron_containing_anatomy_tags = [
             "Painted_domain",
             "Synaptic_neuropil_domain",
@@ -2684,21 +2687,175 @@ def load_volume(self, template=None, verbose=False, query_by_label=True, force_r
                 self._volume.label = self.name
                 self._volume.id = self.id
 
+    def _extract_url_parameter(self, url, parameter='dataset', verbose=False):
+        """
+        Extract the value of a query parameter from a URL.
+        """
+        from urllib.parse import urlparse, parse_qs
+        # Parse the URL
+        parsed_url = urlparse(url)
+        print("Parsed URL: ", parsed_url) if verbose else None
+        # Extract the query parameters as a dictionary
+        query_params = parse_qs(parsed_url.query)
+        print("Query Params: ", query_params) if verbose else None
+        # Get the value of the 'dataset' parameter
+        param_value = query_params.get(parameter)
+        print("Param Value: ", param_value) if verbose else None
+        # Return the first value if the parameter exists
+        if param_value:
+            print(f"Found {parameter} parameter {param_value} of type {type(param_value)}") if verbose else None
+            if isinstance(param_value, list):
+                print("Returning first value: ", param_value[0]) if verbose else None
+                return param_value[0]
+            if isinstance(param_value, str):
+                print("Returning value: ", param_value) if verbose else None
+                return param_value
+        return None
+
     def load_skeleton_synaptic_connections(self, template=None, verbose=False):
         """
         Load the synaptic connections for the neuron's skeleton.
         """
+        import flybrains
+        import navis
+        from navis import transforms
+        import pymaid
+        from navis.interfaces.neuprint import fetch_synapses, Client
+
+        # Load the correct template
         template = self.get_default_template(template=template)
+
+        # Check if skeleton is loaded, else load it
         if not self._skeleton or self._skeleton_template != template:
-            print(f"No skeleton loaded yet for {self.name} so loading...") if verbose else None
-            template = self.get_default_template(template=template)
+            if verbose:
+                print(f"No skeleton loaded yet for {self.name} so loading...")
             self.load_skeleton(template=template, verbose=verbose)
+
         if self._skeleton:
-            print(f"Loading synaptic connections for {self.name}...") if verbose else None
-            xref = self.xref
-            # TODO: Load synaptic connections
-            # see https://github.com/navis-org/navis/blob/1eead062710af6adabc9e9c40196ad7be029cb52/navis/interfaces/neuprint.py#L491
-        print("FEATURE NOT YET IMPLEMENTED")
+            if verbose:
+                print(f"Loading synaptic connections for {self.name}...")
+
+            if 'catmaid' in self.xref_url:
+                if verbose:
+                    print("Loading synaptic connections from CATMAID...")
+
+                skid = int(self.xref_accession)
+                db = self.xref_id.split(':')[0]
+                server = "https://" + self.xref_url.split('://')[-1].split('/')[0]
+                pid = self._extract_url_parameter(self.xref_url, parameter='pid', verbose=verbose)
+
+                # Connect to CATMAID
+                catmaid = pymaid.connect_catmaid(server=server, project_id=pid, api_token=CATMAID_TOKEN, max_threads=10)
+                connectors = pymaid.get_connectors([skid], remote_instance=catmaid)
+
+                # Get connector details to link node_ids
+                links = pymaid.get_connector_details(connectors.connector_id)
+                connectors['node_id'] = connectors.connector_id.map(links.set_index('connector_id').node_id.to_dict())
+
+                # Perform alignment if needed
+                available_transforms = transforms.registry.summary()
+                source, target = self.determine_transform(db, available_transforms, template)
+
+                if source and target:
+                    if verbose:
+                        print(f"Transforming from {source} to {target}...")
+                    aligned_connectors = navis.xform_brain(connectors, source=source, target=target)
+                else:
+                    aligned_connectors = connectors
+
+                # Attach connectors to the skeleton
+                aligned_connectors['type'] = aligned_connectors['type'].str.lower()  # Ensure 'type' is lowercase
+                self._skeleton._set_connectors(aligned_connectors)
+                if verbose:
+                    print(f"Connectors set for {self.name}")
+                return aligned_connectors
+
+            elif 'neuprint' in self.xref_url:
+                if verbose:
+                    print("Loading synaptic connections from neuprint...")
+
+                bodyId = int(self.xref_accession)
+                db = self.xref_id.split(':')[0]
+                server = self.xref_url.split('://')[-1].split('/')[0]
+                ds = self._extract_url_parameter(self.xref_url, parameter='dataset', verbose=verbose)
+
+                # Connect to neuprint
+                client = Client(server=server, dataset=ds, token=NEUPRINT_TOKEN)
+                connectors = fetch_synapses(bodyId, client=client)
+
+                # Get transforms if needed
+                available_transforms = transforms.registry.summary()
+                source, target = self.determine_transform(ds, available_transforms, template)
+
+                if source and target:
+                    if verbose:
+                        print(f"Transforming from {source} to {target}...")
+                    aligned_connectors = navis.xform_brain(connectors, source=source, target=target)
+                else:
+                    aligned_connectors = connectors
+
+                # Ensure 'connector_id' exists and bodyId gets mapped to 'id'
+                if 'connector_id' not in aligned_connectors.columns:
+                    aligned_connectors['connector_id'] = aligned_connectors.index
+                if 'id' not in aligned_connectors.columns:
+                    acc = aligned_connectors.bodyId.to_list()
+                    vfb_ids = self.vfb.xref_2_vfb_id(acc=acc, db=db, return_just_ids=True, verbose=verbose)
+                    if len(vfb_ids) != len(acc):
+                        print("Some bodyIds could not be mapped to VFB IDs.")
+                    aligned_connectors['id'] = vfb_ids
+
+                # Attach connectors to the skeleton
+                aligned_connectors['type'] = aligned_connectors['type'].str.lower()  # Ensure 'type' is lowercase
+                if verbose:
+                    print(f"Setting connectors for {self.name}...")
+                if verbose:
+                    print("Connectors: ", aligned_connectors)
+                self._skeleton._set_connectors(aligned_connectors)
+                if verbose:
+                    print(f"Connectors set for {self.name}")
+                return aligned_connectors
+
+            else:
+                if verbose:
+                    print("Unknown data source for synaptic connections.")
+                return None
+
+        else:
+            if verbose:
+                print("Skeleton not loaded; cannot load synaptic connections.")
+            return None
+
+    def determine_transform(self, name, available_transforms, template):
+        """
+        Determine the source and target brain space for the transformation.
+        """
+        source = None
+        target = None
+
+        if name.upper() in available_transforms.source.to_list():
+            source = name.upper()
+        elif name.lower() in available_transforms.source.to_list():
+            source = name.lower()
+
+        if self.vfb.lookup_name(template) in available_transforms.target.to_list():
+            target = template
+
+        if not source or not target:
+            if 'fafb' in name.lower():
+                source = 'FAFB'
+                target = 'JRC2018U'
+            elif 'fanc' in name.lower():
+                source = 'FANC'
+                target = 'JRC2018U'
+            elif 'hemibrain' in name.lower():
+                source = 'hemibrain'
+                target = 'JRC2018U'
+            elif 'optic-lobe' in name.lower():
+                source = 'JRCFIB2022Mraw'
+                target = 'JRC2018U'
+
+        return source, target
+
 
 
     def get_default_template(self, template=None):