From 775dc3f2b88a2dbf3577f30def36c3d3f3600d39 Mon Sep 17 00:00:00 2001 From: Tom Davidson Date: Thu, 17 Jan 2019 15:20:40 -0800 Subject: [PATCH 1/5] Replace references to deprecated Clustering with new misc.Units --- docs/gallery/domain/ecephys.py | 16 +++++++++++----- docs/gallery/general/file.py | 10 +++++----- docs/source/overview_nwbfile.rst | 7 +------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/gallery/domain/ecephys.py b/docs/gallery/domain/ecephys.py index 7e8a64ba5..00efef570 100644 --- a/docs/gallery/domain/ecephys.py +++ b/docs/gallery/domain/ecephys.py @@ -86,10 +86,12 @@ # Extracellular recordings # ^^^^^^^^^^^^^^^^^^^^^^^^ # -# The two main classes for storing extracellular recordings are :py:class:`~pynwb.ecephys.ElectricalSeries` +# The main classes for storing extracellular recordings are :py:class:`~pynwb.ecephys.ElectricalSeries` # and :py:class:`~pynwb.ecephys.SpikeEventSeries`. :py:class:`~pynwb.ecephys.ElectricalSeries` should be used # for storing raw voltage traces, local-field potential and filtered voltage traces and -# :py:class:`~pynwb.ecephys.SpikeEventSeries` is meant for storing spike waveforms. +# :py:class:`~pynwb.ecephys.SpikeEventSeries` is meant for storing spike waveforms (typically in preparation for +# clustering). The results of spike clustering (e.g. per-unit metadata and spike times) should be stored in the +# top-level :py:class:`~pynwb.misc.Units` table. # # In addition to the *data* and *timestamps* fields inherited # from :py:class:`~pynwb.base.TimeSeries` class, these two classs will require metadata about the elctrodes @@ -154,13 +156,17 @@ # using these objects. # # For storing spike data, there are two options. Which one you choose depends on what data you have available. -# If you need to store the raw voltage traces, you should store your the traces with +# If you need to store the complete, continuous raw voltage traces, you should store your the traces with # :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition ` data, and use # the :py:class:`~pynwb.ecephys.EventDetection` class for identifying the spike events in your raw traces. -# If you do not want to store the raw voltage traces and only the spike events, you should use -# the :py:class:`~pynwb.ecephys.EventWaveform` class, which can store one or more +# If you do not want to store the raw voltage traces and only the waveform 'snippets' surrounding spike events, +# you should use the :py:class:`~pynwb.ecephys.EventWaveform` class, which can store one or more # :py:class:`~pynwb.ecephys.SpikeEventSeries` objects. # +# The results of spike sorting (or clustering) should be stored in the top-level :py:class:`~pynwb.misc.Units` table. +# Note that it is not required to store spike waveforms in order to store spike events or waveforms--if you only +# want to store the spike times of clustered units you can use only the Units table. +# # For local field potential data, there are two options. Again, which one you choose depends on what data you # have available. With both options, you should store your traces with :py:class:`~pynwb.ecephys.ElectricalSeries` # objects. If you are storing unfiltered local field potential data, you should store diff --git a/docs/gallery/general/file.py b/docs/gallery/general/file.py index 51d8d9d68..142c1e8f7 100644 --- a/docs/gallery/general/file.py +++ b/docs/gallery/general/file.py @@ -96,11 +96,11 @@ # ------------------ # # *Processing modules* are used for storing a set of data interfaces that are related to a particular -# processing workflow. For example, if you want to store intermediate and final results of a spike sorting workflow, -# you would create a :py:class:`~pynwb.base.ProcessingModule` that contains data interfaces that represent -# the common steps in spike sorting e.g. :py:class:`~pynwb.ecephys.EventDetection`, -# :py:class:`~pynwb.ecephys.EventWaveform`, :py:class:`~pynwb.ecephys.FeatureExtraction`, -# :py:class:`~pynwb.ecephys.Clustering`, :py:class:`~pynwb.ecephys.ClusterWaveform`. +# processing workflow. For example, if you want to store the intermediate results of a spike sorting workflow, +# you could create a :py:class:`~pynwb.base.ProcessingModule` that contains data interfaces that represent +# the common first steps in spike sorting e.g. :py:class:`~pynwb.ecephys.EventDetection`, +# :py:class:`~pynwb.ecephys.EventWaveform`, :py:class:`~pynwb.ecephys.FeatureExtraction`. The final results of +# the sorting could then be stored in the top-level :py:class:`~pynwb.misc.Units` table (see below). # # Processing modules can be created using :py:func:`~pynwb.file.NWBFile.create_processing_module`: diff --git a/docs/source/overview_nwbfile.rst b/docs/source/overview_nwbfile.rst index 7558d2550..890ed16e8 100644 --- a/docs/source/overview_nwbfile.rst +++ b/docs/source/overview_nwbfile.rst @@ -72,18 +72,13 @@ Processing module objects are unique collections of analysis results. To standar common analyses, NWB provides the concept of an *NWBDataInterface*, where the output of common analyses are represented as objects that extend the :py:class:`~pynwb.core.NWBDataInterface` class. In most cases, you will not need to interact with the :py:class:`~pynwb.core.NWBDataInterface` class directly. -More commonly, you will be creating instances of classes that extend this class. For example, a common -analysis step for spike data (represented in NWB as a :py:class:`~pynwb.ecephys.SpikeEventSeries` object) -is spike clustering. In NWB, the result of this kind of analysis will be represented with a -:py:class:`~pynwb.ecephys.Clustering` object. +More commonly, you will be creating instances of classes that extend this class. The following analysis :py:class:`~pynwb.core.NWBDataInterface` objects are provided by the API and NWB specification: * :py:class:`~pynwb.behavior.BehavioralEpochs` * :py:class:`~pynwb.behavior.BehavioralEvents` * :py:class:`~pynwb.behavior.BehavioralTimeSeries` - * :py:class:`~pynwb.ecephys.ClusterWaveforms` - * :py:class:`~pynwb.ecephys.Clustering` * :py:class:`~pynwb.behavior.CompassDirection` * :py:class:`~pynwb.ophys.DfOverF` * :py:class:`~pynwb.ecephys.EventDetection` From fcb919a3b5c50a92c473e04a6a0b66f7e0204a27 Mon Sep 17 00:00:00 2001 From: Tom Davidson Date: Fri, 18 Jan 2019 00:10:23 -0800 Subject: [PATCH 2/5] flake8 fixes --- docs/gallery/general/file.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/gallery/general/file.py b/docs/gallery/general/file.py index 6e58b0163..e2f33ff65 100644 --- a/docs/gallery/general/file.py +++ b/docs/gallery/general/file.py @@ -198,9 +198,12 @@ nwbfile.add_unit_column('location', 'the anatomical location of this unit') nwbfile.add_unit_column('quality', 'the quality for the inference of this unit') -nwbfile.add_unit(id=1, spike_times=[2.2, 3.0, 4.5], obs_intervals=[[1, 10]], location='CA1', quality=0.95) -nwbfile.add_unit(id=2, spike_times=[2.2, 3.0, 25.0, 26.0], obs_intervals=[[1, 10], [20, 30]], location='CA3', quality=0.85) -nwbfile.add_unit(id=3, spike_times=[1.2, 2.3, 3.3, 4.5], obs_intervals=[[1, 10], [20, 30]], location='CA1', quality=0.90) +nwbfile.add_unit(id=1, spike_times=[2.2, 3.0, 4.5], + obs_intervals=[[1, 10]], location='CA1', quality=0.95) +nwbfile.add_unit(id=2, spike_times=[2.2, 3.0, 25.0, 26.0], + obs_intervals=[[1, 10], [20, 30]], location='CA3', quality=0.85) +nwbfile.add_unit(id=3, spike_times=[1.2, 2.3, 3.3, 4.5], + obs_intervals=[[1, 10], [20, 30]], location='CA1', quality=0.90) #################### # .. _units_fields_ref: From 31cf96ebf26f7e101eeaf9f2d0054af88b5750f9 Mon Sep 17 00:00:00 2001 From: Tom Davidson Date: Fri, 18 Jan 2019 00:18:19 -0800 Subject: [PATCH 3/5] Replace references to Clustering/ClusterWaveforms in docs --- docs/source/extensions.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 8d9ec8491..47ac34fa0 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -123,8 +123,8 @@ Extend an existing type attributes=addl_attributes, datasets=addl_datasets, groups=addl_groups, - neurodata_type_inc='Clustering', - neurodata_type_def='MyExtendedClustering') + neurodata_type_inc='SpikeEventSeries', + neurodata_type_def='MyExtendedSpikeEventSeries') Existing types can be instantiated by specifying `neurodata_type_inc` alone. @@ -139,8 +139,8 @@ Existing types can be instantiated by specifying `neurodata_type_inc` alone. spec = NWBGroupSpec('An extended NWB type', groups=addl_groups, - neurodata_type_inc='Clustering', - neurodata_type_def='MyExtendedClustering') + neurodata_type_inc='SpikeEventSeries', + neurodata_type_def='MyExtendedSpikeEventSeries') Datasets can be extended in the same manner (with regard to `neurodata_type_inc` and `neurodata_type_def`, @@ -169,19 +169,19 @@ Create a new namespace with extensions ns_builder = NWBNamespaceBuilder("Extension for use in my laboratory", "mylab", ...) # create extensions - ext1 = NWBGroupSpec('A custom Clustering interface', + ext1 = NWBGroupSpec('A custom SpikeEventSeries interface', attributes=[...] datasets=[...], groups=[...], - neurodata_type_inc='Clustering', - neurodata_type_def='MyExtendedClustering') + neurodata_type_inc='SpikeEventSeries', + neurodata_type_def='MyExtendedSpikeEventSeries') - ext2 = NWBGroupSpec('A custom ClusterWaveforms interface', + ext2 = NWBGroupSpec('A custom EventDetection interface', attributes=[...] datasets=[...], groups=[...], - neurodata_type_inc='ClusterWaveforms', - neurodata_type_def='MyExtendedClusterWaveforms') + neurodata_type_inc='EventDetection', + neurodata_type_def='MyExtendedEventDetection') # add the extension From 0e2c6fc4ca15d19dfa1405fe3fdb7755fd6d56a8 Mon Sep 17 00:00:00 2001 From: Tom Davidson Date: Fri, 18 Jan 2019 00:18:58 -0800 Subject: [PATCH 4/5] Mark Clustering and ClusterWaveforms as deprecated, add DeprecationWarning --- src/pynwb/data/nwb.ecephys.yaml | 22 +++++++++++----------- src/pynwb/ecephys.py | 8 ++++++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/pynwb/data/nwb.ecephys.yaml b/src/pynwb/data/nwb.ecephys.yaml index eea0cfada..355f6807d 100644 --- a/src/pynwb/data/nwb.ecephys.yaml +++ b/src/pynwb/data/nwb.ecephys.yaml @@ -94,18 +94,18 @@ groups: - null - neurodata_type_def: ClusterWaveforms neurodata_type_inc: NWBDataInterface - doc: 'The mean waveform shape, including standard deviation, of the different clusters. - Ideally, the waveform analysis should be performed on data that is only high-pass - filtered. This is a separate module because it is expected to require updating. - For example, IMEC probes may require different storage requirements to store/display - mean waveforms, requiring a new interface or an extension of this one.' + doc: 'DEPRECATED The mean waveform shape, including standard deviation, of the + different clusters. Ideally, the waveform analysis should be performed on data that + is only high-pass filtered. This is a separate module because it is expected to require + updating. For example, IMEC probes may require different storage requirements to + store/display mean waveforms, requiring a new interface or an extension of this one.' attributes: - name: help dtype: text doc: Value is 'Mean waveform shape of clusters. Waveforms should be high-pass filtered (ie, not the same bandpass filter used waveform analysis and clustering)' - value: Mean waveform shape of clusters. Waveforms should be high-pass filtered - (ie, not the same bandpass filter used waveform analysis and clustering) + value: DEPRECATED Mean waveform shape of clusters. Waveforms should be high-pass + filtered (ie, not the same bandpass filter used waveform analysis and clustering) datasets: - name: waveform_filtering dtype: text @@ -138,15 +138,15 @@ groups: default_name: ClusterWaveforms - neurodata_type_def: Clustering neurodata_type_inc: NWBDataInterface - doc: Clustered spike data, whether from automatic clustering tools (e.g., klustakwik) - or as a result of manual sorting. + doc: DEPRECATED Clustered spike data, whether from automatic clustering tools (e.g., + klustakwik) or as a result of manual sorting. attributes: - name: help dtype: text doc: Value is 'Clustered spike data, whether from automatic clustering tools (eg, klustakwik) or as a result of manual sorting' - value: Clustered spike data, whether from automatic clustering tools (eg, klustakwik) - or as a result of manual sorting + value: DEPRECATED Clustered spike data, whether from automatic clustering tools + (eg, klustakwik) or as a result of manual sorting datasets: - name: description dtype: text diff --git a/src/pynwb/ecephys.py b/src/pynwb/ecephys.py index db7db3b66..dc082e5ee 100644 --- a/src/pynwb/ecephys.py +++ b/src/pynwb/ecephys.py @@ -204,6 +204,7 @@ class EventWaveform(MultiContainerInterface): @register_class('Clustering', CORE_NAMESPACE) class Clustering(NWBDataInterface): """ + DEPRECATED in favor of :py:meth:`~pynwb.misc.Units`. Specifies cluster event times and cluster metric for maximum ratio of waveform peak to RMS on any channel in cluster. """ @@ -215,7 +216,7 @@ class Clustering(NWBDataInterface): 'times' ) - __help = ("Clustered spike data, whether from automatic clustering " + __help = ("[DEPRECATED] Clustered spike data, whether from automatic clustering " "tools (eg, klustakwik) or as a result of manual sorting.") @docval({'name': 'description', 'type': str, @@ -229,6 +230,8 @@ class Clustering(NWBDataInterface): 'shape': (None,)}, {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'Clustering'}) def __init__(self, **kwargs): + import warnings + warnings.warn("use pynwb.misc.Units or NWBFile.units instead", DeprecationWarning) description, num, peak_over_rms, times = popargs( 'description', 'num', 'peak_over_rms', 'times', kwargs) super(Clustering, self).__init__(**kwargs) @@ -241,6 +244,7 @@ def __init__(self, **kwargs): @register_class('ClusterWaveforms', CORE_NAMESPACE) class ClusterWaveforms(NWBDataInterface): """ + DEPRECATED in favor of :py:meth:`~pynwb.misc.Units`. Describe cluster waveforms by mean and standard deviation for at each sample. """ @@ -249,7 +253,7 @@ class ClusterWaveforms(NWBDataInterface): 'waveform_mean', 'waveform_sd') - __help = ("Mean waveform shape of clusters. Waveforms should be " + __help = ("[DEPRECATED] Mean waveform shape of clusters. Waveforms should be " "high-pass filtered (ie, not the same bandpass filter " "used waveform analysis and clustering)") From 2669a736674f9de2d1c167cad5d41b0224be6d6c Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Fri, 18 Jan 2019 00:30:25 -0800 Subject: [PATCH 5/5] More detailed deprecation message for ClusterWaveforms Co-Authored-By: tjd2002 --- src/pynwb/ecephys.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pynwb/ecephys.py b/src/pynwb/ecephys.py index dc082e5ee..c68e23483 100644 --- a/src/pynwb/ecephys.py +++ b/src/pynwb/ecephys.py @@ -244,7 +244,9 @@ def __init__(self, **kwargs): @register_class('ClusterWaveforms', CORE_NAMESPACE) class ClusterWaveforms(NWBDataInterface): """ - DEPRECATED in favor of :py:meth:`~pynwb.misc.Units`. + DEPRECATED. `ClusterWaveforms` was deprecated on Oct 27, 2018 and will be removed in a future release. + Please use the :py:meth:`~pynwb.misc.Units` table to store waveform mean and standard deviation + e.g. `nwbfile.units.add_unit(..., waveform_mean=..., waveform_sd=...)` . Describe cluster waveforms by mean and standard deviation for at each sample. """