Merge pull request #1245 from NeurodataWithoutBorders/schema_2.3.0

Co-authored-by: Ryan Ly <[email protected]> Co-authored-by: Ben Dichter <[email protected]>
NeurodataWithoutBorders · May 18, 2021 · 2934fb0 · 2934fb0
2 parents 130665a + 50eb5b9
commit 2934fb0
Show file tree

Hide file tree

Showing 26 changed files with 336 additions and 68 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -354,7 +354,7 @@ jobs:
           command: |
             python -m venv ../venv
             . ../venv/bin/activate
-            pip install githubrelease
+            pip install "click<8" githubrelease
             githubrelease release $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME \
               create $CIRCLE_TAG --name $CIRCLE_TAG \
               --publish ./dist/*

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,14 +1,28 @@
 # PyNWB Changelog
 
-## PyNWB 1.5.0 (April 23, 2021)
+## PyNWB 1.5.0 (May 17, 2021)
 
 ### New features:
 - `NWBFile.add_scratch(...)` and `ScratchData.__init__(...)` now accept scalar data in addition to the currently
   accepted types. @rly (#1309)
 - Support `pathlib.Path` paths when opening files with `NWBHDF5IO`. @dsleiter (#1314)
 - Use HDMF 2.5.1. See the [HDMF release notes](https://github.com/hdmf-dev/hdmf/releases/tag/2.5.1) for details.
 - Support `driver='ros3'` in `NWBHDF5IO` for streaming NWB files directly from s3. @bendichter (#1331)
-........ TODO
+- Update documentation, CI GitHub processes. @oruebel @yarikoptic, @bendichter, @TomDonoghue, @rly
+  (#1311, #1336, #1351, #1352, #1345, #1340, #1327)
+- Set default `neurodata_type_inc` for `NWBGroupSpec`, `NWBDatasetSpec`. @rly (#1295)
+- Add support for nwb-schema 2.3.0. @rly (#1245, #1330)
+  - Add optional `waveforms` column to the `Units` table.
+  - Add optional `strain` field to `Subject`.
+  - Add to `DecompositionSeries` an optional `DynamicTableRegion` called `source_channels`.
+  - Add to `ImageSeries` an optional link to `Device`.
+  - Add optional `continuity` field to `TimeSeries`.
+  - Add optional `filtering` attribute to `ElectricalSeries`.
+  - Clarify documentation for electrode impedance and filtering.
+  - Set the `stimulus_description` for `IZeroCurrentClamp` to have the fixed value "N/A".
+  - See https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html for full schema release notes.
+- Add support for HDMF 2.5.3. @rly @ajtritt (#1325, #1355, #1360, #1245, #1287)
+  - See https://github.com/hdmf-dev/hdmf/releases for full HDMF release notes.
 
 ## PyNWB 1.4.0 (August 12, 2020)
 

diff --git a/requirements-doc.txt b/requirements-doc.txt
@@ -2,4 +2,4 @@ sphinx
 matplotlib
 sphinx_rtd_theme
 sphinx-gallery
-allensdk
+allensdk  # note that as of allensdk 2.10.0, python 3.8 is not supported
diff --git a/requirements-min.txt b/requirements-min.txt
@@ -1,7 +1,7 @@
 # package dependencies and their minimum versions for installing PyNWB
 # the requirements here specify '==' for testing; setup.py replaces '==' with '>='
 h5py==2.9,<3  # support for setting attrs to lists of utf-8 added in 2.9
-hdmf==2.5.2,<3
+hdmf==2.5.5,<3
 numpy==1.16,<1.21
 pandas==0.23,<2
 python-dateutil==2.7,<3

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 h5py==2.10.0
-hdmf==2.5.2
+hdmf==2.5.5
 numpy==1.18.5
 pandas==0.25.3
 python-dateutil==2.8.1

diff --git a/src/pynwb/base.py b/src/pynwb/base.py
@@ -92,7 +92,8 @@ class TimeSeries(NWBDataInterface):
                      "rate",
                      "starting_time_unit",
                      "control",
-                     "control_description")
+                     "control_description",
+                     "continuity")
 
     __time_unit = "seconds"
 
@@ -119,18 +120,28 @@ class TimeSeries(NWBDataInterface):
             {'name': 'control', 'type': Iterable, 'doc': 'Numerical labels that apply to each element in data',
              'default': None},
             {'name': 'control_description', 'type': Iterable, 'doc': 'Description of each control value',
-             'default': None})
+             'default': None},
+            {'name': 'continuity', 'type': str, 'default': None, 'enum': ["continuous", "instantaneous", "step"],
+             'doc': 'Optionally describe the continuity of the data. Can be "continuous", "instantaneous", or'
+                    '"step". For example, a voltage trace would be "continuous", because samples are recorded from a '
+                    'continuous process. An array of lick times would be "instantaneous", because the data represents '
+                    'distinct moments in time. Times of image presentations would be  "step" because the picture '
+                    'remains the same until the next time-point. This field is optional, but is useful in providing '
+                    'information about the underlying data. It may inform the way this data is interpreted, the way it '
+                    'is visualized, and what analysis methods are applicable.'})
     def __init__(self, **kwargs):
         """Create a TimeSeries object
         """
+
         call_docval_func(super(TimeSeries, self).__init__, kwargs)
         keys = ("resolution",
                 "comments",
                 "description",
                 "conversion",
                 "unit",
                 "control",
-                "control_description")
+                "control_description",
+                "continuity")
         for key in keys:
             val = kwargs.get(key)
             if val is not None:

diff --git a/src/pynwb/ecephys.py b/src/pynwb/ecephys.py
@@ -49,7 +49,8 @@ class ElectricalSeries(TimeSeries):
 
     __nwbfields__ = ({'name': 'electrodes', 'required_name': 'electrodes',
                       'doc': 'the electrodes that generated this electrical series', 'child': True},
-                     'channel_conversion')
+                     'channel_conversion',
+                     'filtering')
 
     @docval(*get_docval(TimeSeries.__init__, 'name'),  # required
             {'name': 'data', 'type': ('array_data', 'data', TimeSeries),  # required
@@ -66,13 +67,21 @@ class ElectricalSeries(TimeSeries):
              "to support the storage of electrical recordings as native values generated by data acquisition systems. "
              "If this dataset is not present, then there is no channel-specific conversion factor, i.e. it is 1 for all"
              " channels.", 'default': None},
+            {'name': 'filtering', 'type': str, 'doc':
+             "Filtering applied to all channels of the data. For example, if this ElectricalSeries represents "
+             "high-pass-filtered data (also known as AP Band), then this value could be 'High-pass 4-pole Bessel "
+             "filter at 500 Hz'. If this ElectricalSeries represents low-pass-filtered LFP data and the type of "
+             "filter is unknown, then this value could be 'Low-pass filter at 300 Hz'. If a non-standard filter "
+             "type is used, provide as much detail about the filter properties as possible.", 'default': None},
             *get_docval(TimeSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate',
                         'comments', 'description', 'control', 'control_description'))
     def __init__(self, **kwargs):
-        name, electrodes, data, channel_conversion = popargs('name', 'electrodes', 'data', 'channel_conversion', kwargs)
+        name, electrodes, data, channel_conversion, filtering = popargs('name', 'electrodes', 'data',
+                                                                        'channel_conversion', 'filtering', kwargs)
         super(ElectricalSeries, self).__init__(name, data, 'volts', **kwargs)
         self.electrodes = electrodes
         self.channel_conversion = channel_conversion
+        self.filtering = filtering
 
 
 @register_class('SpikeEventSeries', CORE_NAMESPACE)

diff --git a/src/pynwb/file.py b/src/pynwb/file.py
@@ -44,18 +44,33 @@ class Subject(NWBContainer):
         'species',
         'subject_id',
         'weight',
-        'date_of_birth'
+        'date_of_birth',
+        'strain'
     )
 
-    @docval({'name': 'age', 'type': str, 'doc': 'the age of the subject', 'default': None},
-            {'name': 'description', 'type': str, 'doc': 'a description of the subject', 'default': None},
-            {'name': 'genotype', 'type': str, 'doc': 'the genotype of the subject', 'default': None},
-            {'name': 'sex', 'type': str, 'doc': 'the sex of the subject', 'default': None},
-            {'name': 'species', 'type': str, 'doc': 'the species of the subject', 'default': None},
-            {'name': 'subject_id', 'type': str, 'doc': 'a unique identifier for the subject', 'default': None},
-            {'name': 'weight', 'type': str, 'doc': 'the weight of the subject', 'default': None},
+    @docval({'name': 'age', 'type': str,
+             'doc': ('The age of the subject. The ISO 8601 Duration format is recommended, e.g., "P90D" for '
+                     '90 days old.'), 'default': None},
+            {'name': 'description', 'type': str,
+             'doc': 'A description of the subject, e.g., "mouse A10".', 'default': None},
+            {'name': 'genotype', 'type': str,
+             'doc': 'The genotype of the subject, e.g., "Sst-IRES-Cre/wt;Ai32(RCL-ChR2(H134R)_EYFP)/wt".',
+             'default': None},
+            {'name': 'sex', 'type': str,
+             'doc': ('The sex of the subject. Using "F" (female), "M" (male), "U" (unknown), or "O" (other) '
+                     'is recommended.'), 'default': None},
+            {'name': 'species', 'type': str,
+             'doc': ('The species of the subject. The formal latin binomal name is recommended, e.g., "Mus musculus"'),
+             'default': None},
+            {'name': 'subject_id', 'type': str, 'doc': 'A unique identifier for the subject, e.g., "A10"',
+             'default': None},
+            {'name': 'weight', 'type': (float, str),
+             'doc': ('The weight of the subject, including units. Using kilograms is recommended. e.g., "0.02 kg". '
+                     'If a float is provided, then the weight will be stored as "[value] kg".'),
+             'default': None},
             {'name': 'date_of_birth', 'type': datetime, 'default': None,
-             'doc': 'datetime of date of birth. May be supplied instead of age.'})
+             'doc': 'The datetime of the date of birth. May be supplied instead of age.'},
+            {'name': 'strain', 'type': str, 'doc': 'The strain of the subject, e.g., "C57BL/6J"', 'default': None})
     def __init__(self, **kwargs):
         kwargs['name'] = 'subject'
         call_docval_func(super(Subject, self).__init__, kwargs)
@@ -65,7 +80,11 @@ def __init__(self, **kwargs):
         self.sex = getargs('sex', kwargs)
         self.species = getargs('species', kwargs)
         self.subject_id = getargs('subject_id', kwargs)
-        self.weight = getargs('weight', kwargs)
+        weight = getargs('weight', kwargs)
+        if isinstance(weight, float):
+            weight = str(weight) + ' kg'
+        self.weight = weight
+        self.strain = getargs('strain', kwargs)
         date_of_birth = getargs('date_of_birth', kwargs)
         if date_of_birth and date_of_birth.tzinfo is None:
             self.date_of_birth = _add_missing_timezone(date_of_birth)

diff --git a/src/pynwb/icephys.py b/src/pynwb/icephys.py
@@ -71,7 +71,7 @@ class PatchClampSeries(TimeSeries):
              'doc': 'IntracellularElectrode group that describes the electrode that was used to apply '
                      'or record this data.'},
             {'name': 'gain', 'type': 'float', 'doc': 'Units: Volt/Amp (v-clamp) or Volt/Volt (c-clamp)'},  # required
-            {'name': 'stimulus_description', 'type': str, 'doc': 'the stimulus name/protocol', 'default': "NA"},
+            {'name': 'stimulus_description', 'type': str, 'doc': 'the stimulus name/protocol', 'default': "N/A"},
             *get_docval(TimeSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate',
                         'comments', 'description', 'control', 'control_description'),
             {'name': 'sweep_number', 'type': (int, 'uint32', 'uint64'),
@@ -138,17 +138,37 @@ class IZeroClampSeries(CurrentClampSeries):
 
     @docval(*get_docval(CurrentClampSeries.__init__, 'name', 'data', 'electrode'),  # required
             {'name': 'gain', 'type': 'float', 'doc': 'Units: Volt/Volt'},  # required
-            *get_docval(CurrentClampSeries.__init__, 'stimulus_description', 'resolution', 'conversion', 'timestamps',
+            {'name': 'stimulus_description', 'type': str,
+             'doc': ('The stimulus name/protocol. Setting this to a value other than "N/A" is deprecated as of '
+                     'NWB 2.3.0.'),
+             'default': 'N/A'},
+            *get_docval(CurrentClampSeries.__init__, 'resolution', 'conversion', 'timestamps',
                         'starting_time', 'rate', 'comments', 'description', 'control', 'control_description',
                         'sweep_number'),
             {'name': 'unit', 'type': str, 'doc': "The base unit of measurement (must be 'volts')",
              'default': 'volts'})
     def __init__(self, **kwargs):
         name, data, electrode, gain = popargs('name', 'data', 'electrode', 'gain', kwargs)
         bias_current, bridge_balance, capacitance_compensation = (0.0, 0.0, 0.0)
+        stimulus_description = popargs('stimulus_description', kwargs)
+        stimulus_description = self._ensure_stimulus_description(name, stimulus_description, 'N/A', '2.3.0')
+        kwargs['stimulus_description'] = stimulus_description
         super().__init__(name, data, electrode, gain, bias_current, bridge_balance, capacitance_compensation,
                          **kwargs)
 
+    def _ensure_stimulus_description(self, name, current_stim_desc, stim_desc, nwb_version):
+        """A helper to ensure correct stimulus_description used.
+
+        Issues a warning with details if `current_stim_desc` is to be ignored, and
+        `stim_desc` to be used instead.
+        """
+        if current_stim_desc != stim_desc:
+            warnings.warn(
+                "Stimulus description '%s' for %s '%s' is ignored and will be set to '%s' "
+                "as per NWB %s."
+                % (current_stim_desc, self.__class__.__name__, name, stim_desc, nwb_version))
+        return stim_desc
+
 
 @register_class('CurrentClampStimulusSeries', CORE_NAMESPACE)
 class CurrentClampStimulusSeries(PatchClampSeries):

diff --git a/src/pynwb/image.py b/src/pynwb/image.py
@@ -5,6 +5,7 @@
 
 from . import register_class, CORE_NAMESPACE
 from .base import TimeSeries, Image
+from .device import Device
 
 
 @register_class('ImageSeries', CORE_NAMESPACE)
@@ -17,7 +18,8 @@ class ImageSeries(TimeSeries):
     __nwbfields__ = ('dimension',
                      'external_file',
                      'starting_frame',
-                     'format')
+                     'format',
+                     'device')
 
     @docval(*get_docval(TimeSeries.__init__, 'name'),  # required
             {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ([None] * 3, [None] * 4),
@@ -40,10 +42,12 @@ class ImageSeries(TimeSeries):
             {'name': 'dimension', 'type': Iterable,
              'doc': 'Number of pixels on x, y, (and z) axes.', 'default': None},
             *get_docval(TimeSeries.__init__, 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate',
-                        'comments', 'description', 'control', 'control_description'))
+                        'comments', 'description', 'control', 'control_description'),
+            {'name': 'device', 'type': Device,
+             'doc': 'Device used to capture the images/video.', 'default': None},)
     def __init__(self, **kwargs):
-        bits_per_pixel, dimension, external_file, starting_frame, format = popargs(
-            'bits_per_pixel', 'dimension', 'external_file', 'starting_frame', 'format', kwargs)
+        bits_per_pixel, dimension, external_file, starting_frame, format, device = popargs(
+            'bits_per_pixel', 'dimension', 'external_file', 'starting_frame', 'format', 'device', kwargs)
         call_docval_func(super(ImageSeries, self).__init__, kwargs)
         if external_file is None and self.data is None:
             raise ValueError("Must supply either external_file or data to %s '%s'."
@@ -56,6 +60,7 @@ def __init__(self, **kwargs):
         else:
             self.starting_frame = None
         self.format = format
+        self.device = device
 
     @property
     def bits_per_pixel(self):
@@ -111,7 +116,11 @@ class ImageMaskSeries(ImageSeries):
              'doc': 'Link to ImageSeries that mask is applied to.'},
             *get_docval(ImageSeries.__init__, 'format', 'external_file', 'starting_frame', 'bits_per_pixel',
                         'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments',
-                        'description', 'control', 'control_description'))
+                        'description', 'control', 'control_description'),
+            {'name': 'device', 'type': Device,
+             'doc': ('Device used to capture the mask data. This field will likely not be needed. '
+                     'The device used to capture the masked ImageSeries data should be stored in the ImageSeries.'),
+             'default': None},)
     def __init__(self, **kwargs):
         masked_imageseries = popargs('masked_imageseries', kwargs)
         super(ImageMaskSeries, self).__init__(**kwargs)
@@ -146,7 +155,7 @@ class OpticalSeries(ImageSeries):
                     'Must also specify frame of reference.'},
             *get_docval(ImageSeries.__init__, 'external_file', 'starting_frame', 'bits_per_pixel',
                         'dimension', 'resolution', 'conversion', 'timestamps', 'starting_time', 'rate', 'comments',
-                        'description', 'control', 'control_description'))
+                        'description', 'control', 'control_description', 'device'))
     def __init__(self, **kwargs):
         distance, field_of_view, orientation = popargs('distance', 'field_of_view', 'orientation', kwargs)
         super(OpticalSeries, self).__init__(**kwargs)

diff --git a/src/pynwb/io/base.py b/src/pynwb/io/base.py
@@ -29,6 +29,7 @@ def __init__(self, spec):
         self.map_spec('unit', data_spec.get_attribute('unit'))
         self.map_spec('resolution', data_spec.get_attribute('resolution'))
         self.map_spec('conversion', data_spec.get_attribute('conversion'))
+        self.map_spec('continuity', data_spec.get_attribute('continuity'))
 
         timestamps_spec = self.spec.get_dataset('timestamps')
         self.map_spec('timestamps_unit', timestamps_spec.get_attribute('unit'))