From 1ee93a96fa84c26cda5484debaca2624cac6fca9 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Jul 2021 02:46:07 -0700 Subject: [PATCH 1/9] Minimal setup for using sphinx.ext.autosummary for the API-docs --- .gitignore | 3 +++ docs/Makefile | 2 +- docs/source/api_docs.rst | 18 ++++-------------- docs/source/conf.py | 4 ++++ src/hdmf/__init__.py | 5 +++++ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 9f3a8f19f..bb4bfc678 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ manifest.json # Auto-generated tutorials /docs/source/tutorials/ +# Auto-generated from sphinx.ext.autosummary +/docs/source/_autosummary + ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/docs/Makefile b/docs/Makefile index 3ef40769e..e86ea849d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -45,7 +45,7 @@ help: @echo " apidoc to build RST from source code" clean: - -rm -rf $(BUILDDIR)/* $(RSTDIR)/$(PKGNAME)*.rst $(RSTDIR)/tutorials + -rm -rf $(BUILDDIR)/* $(RSTDIR)/$(PKGNAME)*.rst $(RSTDIR)/tutorials $(RSTDIR)/_autosummary html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/source/api_docs.rst b/docs/source/api_docs.rst index 119c86401..35ea9af2e 100644 --- a/docs/source/api_docs.rst +++ b/docs/source/api_docs.rst @@ -4,20 +4,10 @@ API Documentation ================== -.. toctree:: - :maxdepth: 2 - :caption: HDMF Modules - - Common data types - Base container classes - Build layer - Specification layer - I/O layer - Data I/O utilities - Development utilities - Validation utilities - Testing utilities - Full list of HDMF package contents +.. autosummary:: + :toctree: _autosummary + :recursive: + hdmf :ref:`modindex` diff --git a/docs/source/conf.py b/docs/source/conf.py index 8af653329..bbba22db3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -50,6 +50,7 @@ # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx_gallery.gen_gallery' @@ -77,6 +78,9 @@ 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), } +# Enable sphinx.ext.autosummary to generate automatic summary for pynwb API docs +autosummary_generate = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 7d9298fe0..a563227d1 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -1,3 +1,8 @@ +""" +The Hierarchical Data Modeling Framework, or HDMF, is a Python package for working with hierarchical data. +It provides APIs for specifying data models, reading and writing data to different storage backends, +and representing data with Python object. +""" from . import query # noqa: F401 from .container import Container, Data, DataRegion from .utils import docval, getargs From 9ae885ea031bee74287f473c5147c31f8bb27044 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 23 Jul 2021 00:22:19 -0700 Subject: [PATCH 2/9] Improve main index for the api docs --- docs/source/api_docs.rst | 96 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/docs/source/api_docs.rst b/docs/source/api_docs.rst index 35ea9af2e..502baa5d8 100644 --- a/docs/source/api_docs.rst +++ b/docs/source/api_docs.rst @@ -4,10 +4,104 @@ API Documentation ================== +.. include:: _autosummary/hdmf.rst + + +Common Data Structures +====================== + +The `hdmf-common-schema `_ defines a collection of reusable data +structures---e.g., :py:class:`~hdfm.common.table.DynamicTable` for extensible data tables---that are useful +for defining new data standards. The following modules define the user API for these common data types +defined in `hdmf-common-schema`_ + +.. autosummary:: + :toctree: _autosummary + :recursive: + + hdmf.common + hdmf.common.io + +I/O +==== + +The following modules define I/O backend functionality for reading/writing data. HDMF defines with +:py:class:`~hdmf.backends.io.HDMFIO` a generic, extensible interface for I/O backends. +:py:class:~hdmf.backends.io.hdf5.h5tools.HDF5IO` then implements :py:class:`~hdmf.backends.io.HDMFIO` to +support storing HDMF data in HDF5. HDMFs decouples the I/O backends from the data API and format schema. +I.e., an I/O backend only needs to be able to interact with :py:mod:`~hdmf.build.builders` describing +common data primitives (e.g., Groups, Datasets, Attributes etc.), but does not need to know anything +about the specifics of a particiular data standard. + +.. autosummary:: + :toctree: _autosummary + :recursive: + + hdmf.backends + hdmf.backends.hdf5 + +Data translation +================ + +The role of the data translation is to translate and integrate data between data APIs (i.e., Containers), +format schema, and data I.O. Given a data Container the data translation generates :py:mod:`~hdmf.build.builders` +for I/O using the relevant :py:mod:`~hdmf.build.objectmapper` to translate attributes from the container +to definitions in a format schema. Conversely, given a collection of :py:mod:`~hdmf.build.builders`, the +data translation will construct corresponding frontend :py:mod:`~hdmf.container` using +:py:mod:`~hdmf.build.objectmapper` to translate fields in the format schema to :py:mod:`~hdmf.container` attributes. +The :py:mod:`~hdmf.build.manager` then defines functionality to manage the build process to translate containers +to builders and vice-versa. + +.. autosummary:: + :toctree: _autosummary + :recursive: + + hdmf.build + +Data Format Specification +========================= + +The following modules define data structures and functionality for specifying format schema +:py:mod:`~hdmf.spec.spec`, managing and reading collections of schema definitions via :py:mod:`~hdmf.spec.namespace` +and :py:mod:`~hdmf.spec.catalog`, and writing schema via :py:mod:`~hdmf.spec.write`. + +.. autosummary:: + :toctree: _autosummary + :recursive: + + hdmf.spec + +Data API +======== + +The modules contained directly at the main hdmf level are used to create user APIs for data standards and +to enhance interaction with data. + .. autosummary:: :toctree: _autosummary :recursive: hdmf + hdmf.container + hdmf.utils + hdmf.data_utils + hdmf.array + hdmf.region + hdmf.monitor + hdmf.query + hdmf.region + +Testing and Validation Utilities +================================ + +Components used to facilitate the definition of unit tests, validation of data files, and validation of format schema. + +.. autosummary:: + :toctree: _autosummary + :recursive: + + hdmf.validate + hdmf.testing + -:ref:`modindex` +:ref:`modindex` \ No newline at end of file From 503ed266ffe44c453aac2d3ad12bdb8a48ffd370 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 23 Jul 2021 00:45:47 -0700 Subject: [PATCH 3/9] Add module docstrings from hdmf.backends and hdmf.common --- src/hdmf/backends/__init__.py | 1 + src/hdmf/backends/hdf5/__init__.py | 1 + src/hdmf/common/__init__.py | 4 ++-- src/hdmf/common/io/__init__.py | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/hdmf/backends/__init__.py b/src/hdmf/backends/__init__.py index 030a2daee..0f29afdc0 100644 --- a/src/hdmf/backends/__init__.py +++ b/src/hdmf/backends/__init__.py @@ -1 +1,2 @@ +"""Package for defining new I/O backends for HDMF and using existing backends, e.g., for HDF5. """ from . import hdf5 diff --git a/src/hdmf/backends/hdf5/__init__.py b/src/hdmf/backends/hdf5/__init__.py index 6abfc8c85..faa9e6207 100644 --- a/src/hdmf/backends/hdf5/__init__.py +++ b/src/hdmf/backends/hdf5/__init__.py @@ -1,3 +1,4 @@ +"""HDF5 backend for reading and writing HDMF data files""" from . import h5_utils, h5tools from .h5_utils import H5RegionSlicer, H5DataIO from .h5tools import HDF5IO, H5SpecWriter, H5SpecReader diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index b906fe7e7..78d068e15 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -1,5 +1,5 @@ -'''This package will contain functions, classes, and objects -for reading and writing data in according to the HDMF-common specification +''' +Package for reading and writing data in accordance with the HDMF-common-schema specification ''' import os.path from copy import deepcopy diff --git a/src/hdmf/common/io/__init__.py b/src/hdmf/common/io/__init__.py index 17b9ef057..62a4cc220 100644 --- a/src/hdmf/common/io/__init__.py +++ b/src/hdmf/common/io/__init__.py @@ -1,3 +1,6 @@ +''' +Object mapper definitions for mapping hdmf-common Containers to Builders and vice-verse +''' from . import multi # noqa: F401 from . import resources # noqa: F401 from . import table # noqa: F401 From 9de098d680a9a1e2c4eeb5defa4f98e69dfed3bd Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 23 Jul 2021 15:15:25 -0700 Subject: [PATCH 4/9] Add missing module and package docstrings used for the auto summary --- src/hdmf/__init__.py | 1 + src/hdmf/backends/hdf5/h5_utils.py | 1 + src/hdmf/backends/hdf5/h5tools.py | 2 ++ src/hdmf/backends/io.py | 11 +++++++++++ src/hdmf/backends/warnings.py | 2 ++ src/hdmf/build/__init__.py | 1 + src/hdmf/build/builders.py | 4 ++++ src/hdmf/build/classgenerator.py | 1 + src/hdmf/build/errors.py | 3 +++ src/hdmf/build/manager.py | 1 + src/hdmf/build/map.py | 1 + src/hdmf/build/objectmapper.py | 2 ++ src/hdmf/build/warnings.py | 4 ++++ src/hdmf/common/__init__.py | 12 +++++++++++- src/hdmf/common/io/alignedtable.py | 1 + src/hdmf/common/io/multi.py | 1 + src/hdmf/common/io/resources.py | 1 + src/hdmf/common/io/table.py | 1 + src/hdmf/common/multi.py | 1 + src/hdmf/common/resources.py | 1 + src/hdmf/common/sparse.py | 1 + src/hdmf/container.py | 1 + src/hdmf/data_utils.py | 1 + src/hdmf/region.py | 1 + src/hdmf/spec/__init__.py | 1 + src/hdmf/spec/catalog.py | 1 + src/hdmf/spec/namespace.py | 7 +++++-- src/hdmf/spec/spec.py | 21 ++++++++++++++++----- src/hdmf/spec/write.py | 5 +++++ src/hdmf/testing/__init__.py | 1 + src/hdmf/utils.py | 3 +++ src/hdmf/validate/__init__.py | 1 + 32 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index a563227d1..b183c8ea8 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -14,6 +14,7 @@ {'name': 'region', 'type': None, 'doc': 'the region reference to use to slice'}, is_method=False) def get_region_slicer(**kwargs): + """get_region_slicer is deprecated and will be removed in HDMF 3.0""" import warnings # noqa: E402 warnings.warn('get_region_slicer is deprecated and will be removed in HDMF 3.0.', DeprecationWarning) diff --git a/src/hdmf/backends/hdf5/h5_utils.py b/src/hdmf/backends/hdf5/h5_utils.py index 82eb9dcff..4f701f54f 100644 --- a/src/hdmf/backends/hdf5/h5_utils.py +++ b/src/hdmf/backends/hdf5/h5_utils.py @@ -1,3 +1,4 @@ +"""Collection of utility classes used for HDF5 I/O, e.g., HDF5 dataset wrapper classes to support lazy read.""" import json import os import warnings diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 9567a8054..709676d14 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -1,3 +1,4 @@ +"""Module implementing the main HDF5IO HDF5 backend for HDMF""" import logging import os.path import warnings @@ -31,6 +32,7 @@ class HDF5IO(HDMFIO): + """HDMF I/O backend for reading and writing HDMF builders to/from HDF5 files.""" __ns_spec_path = 'namespace' # path to the namespace dataset within a namespace group diff --git a/src/hdmf/backends/io.py b/src/hdmf/backends/io.py index d5fb6a889..cec485b52 100644 --- a/src/hdmf/backends/io.py +++ b/src/hdmf/backends/io.py @@ -1,3 +1,4 @@ +"""Module defining the base classes for defining an HDMF I/O backend""" from abc import ABCMeta, abstractmethod from pathlib import Path @@ -7,6 +8,16 @@ class HDMFIO(metaclass=ABCMeta): + """ + Base class for implementing a new I/O backend for HDMF + + Derived classes must implement the following abstract methods: + + * :py:meth:`~hdmf.backends.io.HDMFIO.read_builder` + * :py:meth:`~hdmf.backends.io.HDMFIO.write_builder` + * :py:meth:`~hdmf.backends.io.HDMFIO.open` + * :py:meth:`~hdmf.backends.io.HDMFIO.close` + """ @docval({'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None}, {"name": "source", "type": (str, Path), diff --git a/src/hdmf/backends/warnings.py b/src/hdmf/backends/warnings.py index 77a711584..d488c739e 100644 --- a/src/hdmf/backends/warnings.py +++ b/src/hdmf/backends/warnings.py @@ -1,3 +1,5 @@ +"""Collection of warning classes for HDMF I/O""" + class BrokenLinkWarning(UserWarning): """ Raised when a group has a key with a None value. diff --git a/src/hdmf/build/__init__.py b/src/hdmf/build/__init__.py index ea5d21152..0bea066a1 100644 --- a/src/hdmf/build/__init__.py +++ b/src/hdmf/build/__init__.py @@ -1,3 +1,4 @@ +"""Package for managing the data translation process between frontend Containers and I/O Builders based on the schema""" from .builders import Builder, DatasetBuilder, GroupBuilder, LinkBuilder, ReferenceBuilder, RegionBuilder from .classgenerator import CustomClassGenerator, MCIClassGenerator from .errors import (BuildError, OrphanContainerBuildError, ReferenceTargetNotBuiltError, ContainerConfigurationError, diff --git a/src/hdmf/build/builders.py b/src/hdmf/build/builders.py index 72a1bbe4f..34c8053cb 100644 --- a/src/hdmf/build/builders.py +++ b/src/hdmf/build/builders.py @@ -1,3 +1,4 @@ +"""Module defining the Builder datastrucures used to represent data for I/O, e.g., GroupBuilder, DatasetBuilder etc.""" import copy as _copy import itertools as _itertools import posixpath as _posixpath @@ -12,6 +13,7 @@ class Builder(dict, metaclass=ABCMeta): + """Base interface class for all Builder types""" @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'}, {'name': 'parent', 'type': 'Builder', 'doc': 'the parent builder of this Builder', 'default': None}, @@ -74,6 +76,8 @@ def __repr__(self): class BaseBuilder(Builder, metaclass=ABCMeta): + """Base class for Builders that support attributes, e.g,. DatasetBuilder and GroupBuilder """ + __attribute = 'attributes' # self dictionary key for attributes @docval({'name': 'name', 'type': str, 'doc': 'The name of the builder.'}, diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index db4f326d3..8c9862437 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,3 +1,4 @@ +"""Module iplementing functionaltiy for automatically generating Container classes from type Specs""" from copy import deepcopy from datetime import datetime diff --git a/src/hdmf/build/errors.py b/src/hdmf/build/errors.py index c030daa2d..5e937977b 100644 --- a/src/hdmf/build/errors.py +++ b/src/hdmf/build/errors.py @@ -1,3 +1,6 @@ +""" +Collection of errors used during the data translation process to construct builders from containers and vice versa. +""" from .builders import Builder from ..container import AbstractContainer from ..utils import docval, getargs diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index c9bbd16ca..f7935a9f4 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,3 +1,4 @@ +"""Collection of classes used to manage the data translation process, e.g., BuildManager and TypeMap. """ import logging from collections import OrderedDict, deque from copy import copy diff --git a/src/hdmf/build/map.py b/src/hdmf/build/map.py index 92b0c7499..0f6e8a5b9 100644 --- a/src/hdmf/build/map.py +++ b/src/hdmf/build/map.py @@ -1,3 +1,4 @@ +"""DEPRECATED: Classes in map.py should be imported from :py:mod:`~hdmf.build` instead""" # this prevents breaking of code that imports these classes directly from map.py from .manager import Proxy, BuildManager, TypeSource, TypeMap # noqa: F401 from .objectmapper import ObjectMapper # noqa: F401 diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index 070d0187d..d271a2d49 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -1,3 +1,5 @@ +"""Module defining the base ObjectMapper for mapping between Spec objects and AbstractContainer attributes""" + import logging import re import warnings diff --git a/src/hdmf/build/warnings.py b/src/hdmf/build/warnings.py index 3a910ad0b..27b5308d1 100644 --- a/src/hdmf/build/warnings.py +++ b/src/hdmf/build/warnings.py @@ -1,3 +1,7 @@ +""" +Collection of warnings used during the data translation process to construct builders from containers and vice versa. +""" + class BuildWarning(UserWarning): """ Base class for warnings that are raised during the building of a container. diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 78d068e15..a32830945 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -71,6 +71,11 @@ def _dec(cls): def __get_resources(): + """ + Get the available schema resources + + :returns: Dict with information about the available namespace YAML file(s) + """ from pkg_resources import resource_filename from os.path import join __core_ns_file_name = 'namespace.yaml' @@ -81,7 +86,7 @@ def __get_resources(): def _get_resources(): - # LEGACY: Needed to support legacy implementation. + """LEGACY: Needed to support legacy implementation, use :py:meth:`~hdmf.common.__get_resources` instead. """ return __get_resources() @@ -98,6 +103,11 @@ def load_namespaces(**kwargs): def available_namespaces(): + """ + Get the names of the namespaces available in the NamespaceCatalog in the global __TYPE_MAP. + + :return: Tuple of strings with the names of the available namespaces + """ return __TYPE_MAP.namespace_catalog.namespaces diff --git a/src/hdmf/common/io/alignedtable.py b/src/hdmf/common/io/alignedtable.py index 3ff7f8d3f..e98f509ad 100644 --- a/src/hdmf/common/io/alignedtable.py +++ b/src/hdmf/common/io/alignedtable.py @@ -1,3 +1,4 @@ +"""Object mapper definitions for types implemented in :py:mod:`~hdmf.common.alignedtable`""" from .. import register_map from ..alignedtable import AlignedDynamicTable from .table import DynamicTableMap diff --git a/src/hdmf/common/io/multi.py b/src/hdmf/common/io/multi.py index c2493255d..b06279ac9 100644 --- a/src/hdmf/common/io/multi.py +++ b/src/hdmf/common/io/multi.py @@ -1,3 +1,4 @@ +"""Object mapper definitions for types implemented in :py:mod:`~hdmf.common.multi`""" from .. import register_map from ..multi import SimpleMultiContainer from ...build import ObjectMapper diff --git a/src/hdmf/common/io/resources.py b/src/hdmf/common/io/resources.py index f6b7ee589..ad0c1dc1e 100644 --- a/src/hdmf/common/io/resources.py +++ b/src/hdmf/common/io/resources.py @@ -1,3 +1,4 @@ +"""Object mapper definitions for types implemented in :py:mod:`~hdmf.common.resources`""" from .. import register_map from ..resources import ExternalResources, KeyTable, ResourceTable, ObjectTable, ObjectKeyTable, EntityTable from ...build import ObjectMapper diff --git a/src/hdmf/common/io/table.py b/src/hdmf/common/io/table.py index b549c094f..cabe666f6 100644 --- a/src/hdmf/common/io/table.py +++ b/src/hdmf/common/io/table.py @@ -1,3 +1,4 @@ +"""Object mapper definitions for types implemented in :py:mod:`~hdmf.common.table`""" from .. import register_map from ..table import DynamicTable, VectorData, VectorIndex, DynamicTableRegion from ...build import ObjectMapper, BuildManager, CustomClassGenerator diff --git a/src/hdmf/common/multi.py b/src/hdmf/common/multi.py index a2b1d2619..b7489b71c 100644 --- a/src/hdmf/common/multi.py +++ b/src/hdmf/common/multi.py @@ -1,3 +1,4 @@ +"""Module for multi-Container interfaces that usually manage multiple instances of set of particular Container types""" from . import register_class from ..container import Container, Data, MultiContainerInterface from ..utils import docval, call_docval_func, popargs diff --git a/src/hdmf/common/resources.py b/src/hdmf/common/resources.py index 602f3127c..0870a5c9f 100644 --- a/src/hdmf/common/resources.py +++ b/src/hdmf/common/resources.py @@ -1,3 +1,4 @@ +"""Module with data structures for managing links to external resources, e.g., ontologies or persistent identifiers""" import pandas as pd from . import register_class, EXP_NAMESPACE diff --git a/src/hdmf/common/sparse.py b/src/hdmf/common/sparse.py index 9cd600689..c8b24569d 100644 --- a/src/hdmf/common/sparse.py +++ b/src/hdmf/common/sparse.py @@ -1,3 +1,4 @@ +"""Module with data structures for stroring sparse matrices """ import h5py import numpy as np import scipy.sparse as sps diff --git a/src/hdmf/container.py b/src/hdmf/container.py index a7d56dd64..8a1f7e1bf 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -1,3 +1,4 @@ +"""Module with base classes for creating frontend data APIs based on the concept of Containers""" import types from abc import abstractmethod from collections import OrderedDict diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 93856a040..a32d96517 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -1,3 +1,4 @@ +"""Module with data utilities, e.g., dataset I/O wrappers for iterative write and for defining I/O properties""" import copy from abc import ABCMeta, abstractmethod from collections.abc import Iterable diff --git a/src/hdmf/region.py b/src/hdmf/region.py index 9feeba401..890a29154 100644 --- a/src/hdmf/region.py +++ b/src/hdmf/region.py @@ -1,3 +1,4 @@ +"""Module with dataset wrapper classes used to support lazy evaluation of data slicing operations on read.""" from abc import ABCMeta, abstractmethod from operator import itemgetter diff --git a/src/hdmf/spec/__init__.py b/src/hdmf/spec/__init__.py index 09ad6d073..c6f250314 100644 --- a/src/hdmf/spec/__init__.py +++ b/src/hdmf/spec/__init__.py @@ -1,3 +1,4 @@ +"""Package for creating format schema, managing collections of scheme definitions, and reading/writing schema.""" from .catalog import SpecCatalog from .namespace import NamespaceCatalog, SpecNamespace, SpecReader from .spec import (AttributeSpec, DatasetSpec, DtypeHelper, DtypeSpec, GroupSpec, LinkSpec, diff --git a/src/hdmf/spec/catalog.py b/src/hdmf/spec/catalog.py index e623aae51..d1d9c0bba 100644 --- a/src/hdmf/spec/catalog.py +++ b/src/hdmf/spec/catalog.py @@ -1,3 +1,4 @@ +"""Catalog for managing a collection of format specifications""" import copy from collections import OrderedDict diff --git a/src/hdmf/spec/namespace.py b/src/hdmf/spec/namespace.py index c60472459..f77603b5a 100644 --- a/src/hdmf/spec/namespace.py +++ b/src/hdmf/spec/namespace.py @@ -1,3 +1,4 @@ +"""Collection of classes for specifying , managing, and reading namespaces, e.g, SpecNamespace, NamespaceCatalog etc.""" import os.path import ruamel.yaml as yaml import string @@ -29,7 +30,7 @@ class SpecNamespace(dict): """ - A namespace for specifications + Specify a namespace for specifications """ __types_key = 'data_types' @@ -173,7 +174,7 @@ def build_namespace(cls, **spec_dict): class SpecReader(metaclass=ABCMeta): - + """Base interface for implementing reader classes for format specifications""" @docval({'name': 'source', 'type': str, 'doc': 'the source from which this reader reads from'}) def __init__(self, **kwargs): self.__source = getargs('source', kwargs) @@ -192,6 +193,7 @@ def read_namespace(self): class YAMLSpecReader(SpecReader): + """Read specification from YAML""" @docval({'name': 'indir', 'type': str, 'doc': 'the path spec files are relative to', 'default': '.'}) def __init__(self, **kwargs): @@ -224,6 +226,7 @@ def __get_spec_path(self, spec_path): class NamespaceCatalog: + """A catalog for storing multiple Namespaces""" @docval({'name': 'group_spec_cls', 'type': type, 'doc': 'the class to use for group specifications', 'default': GroupSpec}, diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index c892cabf5..48271fd64 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -1,3 +1,4 @@ +"""Collection of classes for representing and creating format specifications, e.g. GroupSpec, DatasetSpec etc.""" import re from abc import ABCMeta from collections import OrderedDict @@ -19,11 +20,19 @@ class DtypeHelper: - # Dict where the keys are the primary data type and the values are list of strings with synonyms for the dtype - # make sure keys are consistent between hdmf.spec.spec.DtypeHelper.primary_dtype_synonyms, - # hdmf.build.objectmapper.ObjectMapper.__dtypes, hdmf.build.manager.TypeMap._spec_dtype_map, - # hdmf.validate.validator.__allowable, and backend dtype maps - # see https://hdmf-schema-language.readthedocs.io/en/latest/description.html#dtype + """ + Utility class with helper functions and attributes used when specifying data types. + + This class is usually not instantiated but defines a collection of static helpers. + """ + + """ + Dict where the keys are the primary data type and the values are list of strings with synonyms for the dtype + make sure keys are consistent between hdmf.spec.spec.DtypeHelper.primary_dtype_synonyms, + hdmf.build.objectmapper.ObjectMapper.__dtypes, hdmf.build.manager.TypeMap._spec_dtype_map, + hdmf.validate.validator.__allowable, and backend dtype maps + see https://hdmf-schema-language.readthedocs.io/en/latest/description.html#dtype + """ primary_dtype_synonyms = { 'float': ["float", "float32"], 'double': ["double", "float64"], @@ -176,6 +185,7 @@ def path(self): class RefSpec(ConstructableDict): + """Specification for object- and region-reference data types used with DatasetSpec and AttributedSpec""" __allowable_types = ('object', 'region') @docval(*_ref_args) @@ -794,6 +804,7 @@ def build_const_args(cls, spec_dict): class LinkSpec(Spec): + """Specification for Links""" @docval(*_link_args) def __init__(self, **kwargs): diff --git a/src/hdmf/spec/write.py b/src/hdmf/spec/write.py index 3725f6781..510f5bdea 100644 --- a/src/hdmf/spec/write.py +++ b/src/hdmf/spec/write.py @@ -1,3 +1,8 @@ +""" +Collection of classes to export format specifications to YAML/JSON + +For reading namespace from file see :py:mod:`~hdmf.spec.namespace` +""" import copy import json import os.path diff --git a/src/hdmf/testing/__init__.py b/src/hdmf/testing/__init__.py index 2d261763f..b945081b7 100644 --- a/src/hdmf/testing/__init__.py +++ b/src/hdmf/testing/__init__.py @@ -1,2 +1,3 @@ +"""Package with utility functions and classes to facilitate the creation of tests""" from .testcase import TestCase, H5RoundTripMixin # noqa: F401 from .utils import remove_test_file # noqa: F401 diff --git a/src/hdmf/utils.py b/src/hdmf/utils.py index 771b0f248..86c72643a 100644 --- a/src/hdmf/utils.py +++ b/src/hdmf/utils.py @@ -1,3 +1,6 @@ +""" +Module with utility functions and classes for creating frontend data functionality (e.g., docval or LabelledDiced) +""" import collections import copy as _copy import types diff --git a/src/hdmf/validate/__init__.py b/src/hdmf/validate/__init__.py index cb515cea8..34212d7e2 100644 --- a/src/hdmf/validate/__init__.py +++ b/src/hdmf/validate/__init__.py @@ -1,3 +1,4 @@ +"""Package defining functionality for validating files generated with HDMF""" from . import errors from .errors import * # noqa: F403 from .validator import ValidatorMap, Validator, AttributeValidator, DatasetValidator, GroupValidator From 736f8047283e3886a0f34ebccdf30ca974f14278 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 23 Jul 2021 15:21:01 -0700 Subject: [PATCH 5/9] Fix falke8 --- src/hdmf/backends/warnings.py | 1 + src/hdmf/build/warnings.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/hdmf/backends/warnings.py b/src/hdmf/backends/warnings.py index d488c739e..c00cf2741 100644 --- a/src/hdmf/backends/warnings.py +++ b/src/hdmf/backends/warnings.py @@ -1,5 +1,6 @@ """Collection of warning classes for HDMF I/O""" + class BrokenLinkWarning(UserWarning): """ Raised when a group has a key with a None value. diff --git a/src/hdmf/build/warnings.py b/src/hdmf/build/warnings.py index 27b5308d1..1a832d16a 100644 --- a/src/hdmf/build/warnings.py +++ b/src/hdmf/build/warnings.py @@ -2,6 +2,7 @@ Collection of warnings used during the data translation process to construct builders from containers and vice versa. """ + class BuildWarning(UserWarning): """ Base class for warnings that are raised during the building of a container. From 9d27372303ad2a87abec09b2e4462742484c6a92 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 29 Jul 2021 09:59:57 -0700 Subject: [PATCH 6/9] Fix make clean on windows --- docs/make.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/make.bat b/docs/make.bat index 9dc52f908..df0775696 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -51,6 +51,7 @@ if "%1" == "clean" ( del /q /s %BUILDDIR%\* del /q %RSTDIR%\%PKGNAME%*.rst rmdir /q /s %RSTDIR%\tutorials + rmdir /q /s %RSTDIR%\_autosummary goto end ) From 252fe55904526c2eba79c2008f261d647c9bb8f8 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 29 Jul 2021 10:00:14 -0700 Subject: [PATCH 7/9] Add autosummary module template with noindex directive --- docs/source/_templates/autosummary/module.rst | 61 +++++++++++++++++++ docs/source/api_docs.rst | 2 +- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 docs/source/_templates/autosummary/module.rst diff --git a/docs/source/_templates/autosummary/module.rst b/docs/source/_templates/autosummary/module.rst new file mode 100644 index 000000000..9e5ccf2f5 --- /dev/null +++ b/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,61 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + :noindex: + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Module Attributes') }} + + .. autosummary:: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/source/api_docs.rst b/docs/source/api_docs.rst index 502baa5d8..aef289335 100644 --- a/docs/source/api_docs.rst +++ b/docs/source/api_docs.rst @@ -104,4 +104,4 @@ Components used to facilitate the definition of unit tests, validation of data f hdmf.testing -:ref:`modindex` \ No newline at end of file +:ref:`modindex` From 1df99ae9ec140d9aea7ecf1e166d9797a5b05fb0 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 25 Mar 2022 17:03:16 -0700 Subject: [PATCH 8/9] Minor update to trigger docs build --- src/hdmf/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index b183c8ea8..49fd5484a 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -14,9 +14,9 @@ {'name': 'region', 'type': None, 'doc': 'the region reference to use to slice'}, is_method=False) def get_region_slicer(**kwargs): - """get_region_slicer is deprecated and will be removed in HDMF 3.0""" + """get_region_slicer is deprecated and will be removed in HDMF 4.0""" import warnings # noqa: E402 - warnings.warn('get_region_slicer is deprecated and will be removed in HDMF 3.0.', DeprecationWarning) + warnings.warn('get_region_slicer is deprecated and will be removed in HDMF 4.0.', DeprecationWarning) dataset, region = getargs('dataset', 'region', kwargs) if isinstance(dataset, (list, tuple, Data)): From 6f5b77d3fdc8d614ed9d1520481c6f6608774ea4 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Sun, 27 Mar 2022 12:59:40 -0700 Subject: [PATCH 9/9] Minor update to trigger RTD PR builds --- src/hdmf/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 49fd5484a..2fb5a25fd 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -1,7 +1,7 @@ """ The Hierarchical Data Modeling Framework, or HDMF, is a Python package for working with hierarchical data. It provides APIs for specifying data models, reading and writing data to different storage backends, -and representing data with Python object. +and representing data with Python objects. """ from . import query # noqa: F401 from .container import Container, Data, DataRegion