Merge pull request #797 from NeurodataWithoutBorders/units-clustering…

…-docs-794

Deprecate Clustering/ClusterWaveform (in favor of misc.Units)

docs/gallery/domain/ecephys.py

@@ -86,10 +86,12 @@
 # Extracellular recordings
 # ^^^^^^^^^^^^^^^^^^^^^^^^
 #
-# The two main classes for storing extracellular recordings are :py:class:`~pynwb.ecephys.ElectricalSeries`
+# The main classes for storing extracellular recordings are :py:class:`~pynwb.ecephys.ElectricalSeries`
 # and :py:class:`~pynwb.ecephys.SpikeEventSeries`. :py:class:`~pynwb.ecephys.ElectricalSeries` should be used
 # for storing raw voltage traces, local-field potential and filtered voltage traces and
-# :py:class:`~pynwb.ecephys.SpikeEventSeries` is meant for storing spike waveforms.
+# :py:class:`~pynwb.ecephys.SpikeEventSeries` is meant for storing spike waveforms (typically in preparation for
+# clustering). The results of spike clustering (e.g. per-unit metadata and spike times) should be stored in the
+# top-level :py:class:`~pynwb.misc.Units` table.
 #
 # In addition to the *data* and *timestamps* fields inherited
 # from :py:class:`~pynwb.base.TimeSeries` class, these two classs will require metadata about the elctrodes
@@ -154,13 +156,17 @@
 # using these objects.
 #
 # For storing spike data, there are two options. Which one you choose depends on what data you have available.
-# If you need to store the raw voltage traces, you should store your the traces with
+# If you need to store the complete, continuous raw voltage traces, you should store your the traces with
 # :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data, and use
 # the :py:class:`~pynwb.ecephys.EventDetection` class for identifying the spike events in your raw traces.
-# If you do not want to store the raw voltage traces and only the spike events, you should use
-# the :py:class:`~pynwb.ecephys.EventWaveform` class, which can store one or more
+# If you do not want to store the raw voltage traces and only the waveform 'snippets' surrounding spike events,
+# you should use the :py:class:`~pynwb.ecephys.EventWaveform` class, which can store one or more
 # :py:class:`~pynwb.ecephys.SpikeEventSeries` objects.
 #
+# The results of spike sorting (or clustering) should be stored in the top-level :py:class:`~pynwb.misc.Units` table.
+# Note that it is not required to store spike waveforms in order to store spike events or waveforms--if you only
+# want to store the spike times of clustered units you can use only the Units table.
+#
 # For local field potential data, there are two options. Again, which one you choose depends on what data you
 # have available. With both options, you should store your traces with :py:class:`~pynwb.ecephys.ElectricalSeries`
 # objects. If you are storing unfiltered local field potential data, you should store

docs/gallery/general/file.py

@@ -96,11 +96,11 @@
 # ------------------
 #
 # *Processing modules* are used for storing a set of data interfaces that are related to a particular
-# processing workflow. For example, if you want to store intermediate and final results of a spike sorting workflow,
-# you would create a :py:class:`~pynwb.base.ProcessingModule` that contains data interfaces that represent
-# the common steps in spike sorting e.g. :py:class:`~pynwb.ecephys.EventDetection`,
-# :py:class:`~pynwb.ecephys.EventWaveform`,  :py:class:`~pynwb.ecephys.FeatureExtraction`,
-# :py:class:`~pynwb.ecephys.Clustering`, :py:class:`~pynwb.ecephys.ClusterWaveform`.
+# processing workflow. For example, if you want to store the intermediate results of a spike sorting workflow,
+# you could create a :py:class:`~pynwb.base.ProcessingModule` that contains data interfaces that represent
+# the common first steps in spike sorting e.g. :py:class:`~pynwb.ecephys.EventDetection`,
+# :py:class:`~pynwb.ecephys.EventWaveform`,  :py:class:`~pynwb.ecephys.FeatureExtraction`. The final results of
+# the sorting could then be stored in the top-level :py:class:`~pynwb.misc.Units` table (see below).
 #
 # Processing modules can be created using :py:func:`~pynwb.file.NWBFile.create_processing_module`:
 
@@ -198,9 +198,12 @@
 nwbfile.add_unit_column('location', 'the anatomical location of this unit')
 nwbfile.add_unit_column('quality', 'the quality for the inference of this unit')
 
-nwbfile.add_unit(id=1, spike_times=[2.2, 3.0, 4.5], obs_intervals=[[1, 10]], location='CA1', quality=0.95)
-nwbfile.add_unit(id=2, spike_times=[2.2, 3.0, 25.0, 26.0], obs_intervals=[[1, 10], [20, 30]], location='CA3', quality=0.85)
-nwbfile.add_unit(id=3, spike_times=[1.2, 2.3, 3.3, 4.5], obs_intervals=[[1, 10], [20, 30]], location='CA1', quality=0.90)
+nwbfile.add_unit(id=1, spike_times=[2.2, 3.0, 4.5],
+                 obs_intervals=[[1, 10]], location='CA1', quality=0.95)
+nwbfile.add_unit(id=2, spike_times=[2.2, 3.0, 25.0, 26.0],
+                 obs_intervals=[[1, 10], [20, 30]], location='CA3', quality=0.85)
+nwbfile.add_unit(id=3, spike_times=[1.2, 2.3, 3.3, 4.5],
+                 obs_intervals=[[1, 10], [20, 30]], location='CA1', quality=0.90)
 
 ####################
 # .. _units_fields_ref:

docs/source/extensions.rst

@@ -123,8 +123,8 @@ Extend an existing type
                         attributes=addl_attributes,
                         datasets=addl_datasets,
                         groups=addl_groups,
-                        neurodata_type_inc='Clustering',
-                        neurodata_type_def='MyExtendedClustering')
+                        neurodata_type_inc='SpikeEventSeries',
+                        neurodata_type_def='MyExtendedSpikeEventSeries')
 
 Existing types can be instantiated by specifying `neurodata_type_inc` alone.
 
@@ -139,8 +139,8 @@ Existing types can be instantiated by specifying `neurodata_type_inc` alone.
 
     spec = NWBGroupSpec('An extended NWB type',
                         groups=addl_groups,
-                        neurodata_type_inc='Clustering',
-                        neurodata_type_def='MyExtendedClustering')
+                        neurodata_type_inc='SpikeEventSeries',
+                        neurodata_type_def='MyExtendedSpikeEventSeries')
 
 
 Datasets can be extended in the same manner (with regard to `neurodata_type_inc` and `neurodata_type_def`,
@@ -169,19 +169,19 @@ Create a new namespace with extensions
     ns_builder = NWBNamespaceBuilder("Extension for use in my laboratory", "mylab", ...)
 
     # create extensions
-    ext1 = NWBGroupSpec('A custom Clustering interface',
+    ext1 = NWBGroupSpec('A custom SpikeEventSeries interface',
                         attributes=[...]
                         datasets=[...],
                         groups=[...],
-                        neurodata_type_inc='Clustering',
-                        neurodata_type_def='MyExtendedClustering')
+                        neurodata_type_inc='SpikeEventSeries',
+                        neurodata_type_def='MyExtendedSpikeEventSeries')
 
-    ext2 = NWBGroupSpec('A custom ClusterWaveforms interface',
+    ext2 = NWBGroupSpec('A custom EventDetection interface',
                         attributes=[...]
                         datasets=[...],
                         groups=[...],
-                        neurodata_type_inc='ClusterWaveforms',
-                        neurodata_type_def='MyExtendedClusterWaveforms')
+                        neurodata_type_inc='EventDetection',
+                        neurodata_type_def='MyExtendedEventDetection')
 
 
     # add the extension

docs/source/overview_nwbfile.rst

@@ -72,18 +72,13 @@ Processing module objects are unique collections of analysis results. To standar
 common analyses, NWB provides the concept of an *NWBDataInterface*, where the output of
 common analyses are represented as objects that extend the :py:class:`~pynwb.core.NWBDataInterface` class.
 In most cases, you will not need to interact with the :py:class:`~pynwb.core.NWBDataInterface` class directly.
-More commonly, you will be creating instances of classes that extend this class. For example, a common
-analysis step for spike data (represented in NWB as a :py:class:`~pynwb.ecephys.SpikeEventSeries` object)
-is spike clustering. In NWB, the result of this kind of analysis will be represented with a
-:py:class:`~pynwb.ecephys.Clustering` object.
+More commonly, you will be creating instances of classes that extend this class.
 
 The following analysis :py:class:`~pynwb.core.NWBDataInterface` objects are provided by the API and NWB specification:
 
   * :py:class:`~pynwb.behavior.BehavioralEpochs`
   * :py:class:`~pynwb.behavior.BehavioralEvents`
   * :py:class:`~pynwb.behavior.BehavioralTimeSeries`
-  * :py:class:`~pynwb.ecephys.ClusterWaveforms`
-  * :py:class:`~pynwb.ecephys.Clustering`
   * :py:class:`~pynwb.behavior.CompassDirection`
   * :py:class:`~pynwb.ophys.DfOverF`
   * :py:class:`~pynwb.ecephys.EventDetection`

src/pynwb/data/nwb.ecephys.yaml

@@ -94,18 +94,18 @@ groups:
     - null
 - neurodata_type_def: ClusterWaveforms
   neurodata_type_inc: NWBDataInterface
-  doc: 'The mean waveform shape, including standard deviation, of the different clusters.
-    Ideally, the waveform analysis should be performed on data that is only high-pass
-    filtered. This is a separate module because it is expected to require updating.
-    For example, IMEC probes may require different storage requirements to store/display
-    mean waveforms, requiring a new interface or an extension of this one.'
+  doc: 'DEPRECATED The mean waveform shape, including standard deviation, of the
+    different clusters. Ideally, the waveform analysis should be performed on data that
+    is only high-pass filtered. This is a separate module because it is expected to require
+    updating. For example, IMEC probes may require different storage requirements to
+    store/display mean waveforms, requiring a new interface or an extension of this one.'
   attributes:
   - name: help
     dtype: text
     doc: Value is 'Mean waveform shape of clusters. Waveforms should be high-pass
       filtered (ie, not the same bandpass filter used waveform analysis and clustering)'
-    value: Mean waveform shape of clusters. Waveforms should be high-pass filtered
-      (ie, not the same bandpass filter used waveform analysis and clustering)
+    value: DEPRECATED Mean waveform shape of clusters. Waveforms should be high-pass
+      filtered (ie, not the same bandpass filter used waveform analysis and clustering)
   datasets:
   - name: waveform_filtering
     dtype: text
@@ -138,15 +138,15 @@ groups:
   default_name: ClusterWaveforms
 - neurodata_type_def: Clustering
   neurodata_type_inc: NWBDataInterface
-  doc: Clustered spike data, whether from automatic clustering tools (e.g., klustakwik)
-    or as a result of manual sorting.
+  doc: DEPRECATED Clustered spike data, whether from automatic clustering tools (e.g.,
+    klustakwik) or as a result of manual sorting.
   attributes:
   - name: help
     dtype: text
     doc: Value is 'Clustered spike data, whether from automatic clustering tools (eg,
       klustakwik) or as a result of manual sorting'
-    value: Clustered spike data, whether from automatic clustering tools (eg, klustakwik)
-      or as a result of manual sorting
+    value: DEPRECATED Clustered spike data, whether from automatic clustering tools
+      (eg, klustakwik) or as a result of manual sorting
   datasets:
   - name: description
     dtype: text

src/pynwb/ecephys.py

@@ -204,6 +204,7 @@ class EventWaveform(MultiContainerInterface):
 @register_class('Clustering', CORE_NAMESPACE)
 class Clustering(NWBDataInterface):
     """
+    DEPRECATED in favor of :py:meth:`~pynwb.misc.Units`.
     Specifies cluster event times and cluster metric for maximum ratio of
     waveform peak to RMS on any channel in cluster.
     """
@@ -215,7 +216,7 @@ class Clustering(NWBDataInterface):
         'times'
     )
 
-    __help = ("Clustered spike data, whether from automatic clustering "
+    __help = ("[DEPRECATED] Clustered spike data, whether from automatic clustering "
               "tools (eg, klustakwik) or as a result of manual sorting.")
 
     @docval({'name': 'description', 'type': str,
@@ -229,6 +230,8 @@ class Clustering(NWBDataInterface):
              'shape': (None,)},
             {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'Clustering'})
     def __init__(self, **kwargs):
+        import warnings
+        warnings.warn("use pynwb.misc.Units or NWBFile.units instead", DeprecationWarning)
         description, num, peak_over_rms, times = popargs(
             'description', 'num', 'peak_over_rms', 'times', kwargs)
         super(Clustering, self).__init__(**kwargs)
@@ -254,7 +257,7 @@ class ClusterWaveforms(NWBDataInterface):
                      'waveform_mean',
                      'waveform_sd')
 
-    __help = ("Mean waveform shape of clusters. Waveforms should be "
+    __help = ("[DEPRECATED] Mean waveform shape of clusters. Waveforms should be "
               "high-pass filtered (ie, not the same bandpass filter "
               "used waveform analysis and clustering)")