Setup sphinx.ext.autosummary for the API-docs

.gitignore

@@ -21,6 +21,9 @@ manifest.json
 # Auto-generated tutorials
 /docs/source/tutorials/
 
+# Auto-generated from sphinx.ext.autosummary
+/docs/source/_autosummary
+
 ### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/

docs/Makefile

@@ -46,7 +46,7 @@ help:
 	@echo "  apidoc     to build RST from source code"
 
 clean:
-	-rm -rf $(BUILDDIR)/* $(RSTDIR)/$(PKGNAME)*.rst $(GALLERYDIR)/*.hdf5 $(GALLERYDIR)/*.sqlite $(RSTDIR)/tutorials
+	-rm -rf $(BUILDDIR)/* $(RSTDIR)/$(PKGNAME)*.rst $(GALLERYDIR)/*.hdf5 $(GALLERYDIR)/*.sqlite $(RSTDIR)/_autosummary $(RSTDIR)/tutorials
 
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html

docs/make.bat

@@ -54,6 +54,7 @@ if "%1" == "clean" (
 	del /q %GALLERYDIR%\*.hdf5
 	del /q %GALLERYDIR%\*.sqlite
 	rmdir /q /s %RSTDIR%\tutorials
+	rmdir /q /s %RSTDIR%\_autosummary
 	goto end
 )
 

docs/source/_templates/autosummary/module.rst

@@ -0,0 +1,61 @@
+{{ fullname | escape | underline}}
+
+.. automodule:: {{ fullname }}
+   :noindex:
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: {{ _('Module Attributes') }}
+
+   .. autosummary::
+   {% for item in attributes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block functions %}
+   {% if functions %}
+   .. rubric:: {{ _('Functions') }}
+
+   .. autosummary::
+   {% for item in functions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block classes %}
+   {% if classes %}
+   .. rubric:: {{ _('Classes') }}
+
+   .. autosummary::
+   {% for item in classes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block exceptions %}
+   {% if exceptions %}
+   .. rubric:: {{ _('Exceptions') }}
+
+   .. autosummary::
+   {% for item in exceptions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+{% block modules %}
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+{% for item in modules %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+{% endblock %}

docs/source/api_docs.rst

@@ -4,20 +4,104 @@
 API Documentation
 ==================
 
-.. toctree::
-   :maxdepth: 2
-   :caption: HDMF Modules
-
-   Common data types <hdmf.common>
-   Base container classes <hdmf.container>
-   Build layer <hdmf.build>
-   Specification layer <hdmf.spec>
-   I/O layer <hdmf.backends>
-   Data I/O utilities <hdmf.data_utils>
-   Development utilities <hdmf.utils>
-   Validation utilities <hdmf.validate>
-   Testing utilities <hdmf.testing>
-   Full list of HDMF package contents <hdmf>
+.. include:: _autosummary/hdmf.rst
+
+
+Common Data Structures
+======================
+
+The `hdmf-common-schema <https://hdmf-common-schema.readthedocs.io/>`_ defines a collection of reusable data
+structures---e.g., :py:class:`~hdfm.common.table.DynamicTable` for extensible data tables---that are useful
+for defining new data standards. The following modules define the user API for these common data types
+defined in `hdmf-common-schema`_
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   hdmf.common
+   hdmf.common.io
+
+I/O
+====
+
+The following modules define I/O backend functionality for reading/writing data. HDMF defines with
+:py:class:`~hdmf.backends.io.HDMFIO` a generic, extensible interface for I/O backends.
+:py:class:~hdmf.backends.io.hdf5.h5tools.HDF5IO` then implements :py:class:`~hdmf.backends.io.HDMFIO` to
+support storing HDMF data in HDF5. HDMFs decouples the  I/O backends from the data API and format schema.
+I.e., an I/O backend only needs to be able to interact with :py:mod:`~hdmf.build.builders` describing
+common data primitives (e.g., Groups, Datasets, Attributes etc.), but does not need to know anything
+about the specifics of a particiular data standard.
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   hdmf.backends
+   hdmf.backends.hdf5
+
+Data translation
+================
+
+The role of the data translation is to translate and integrate data between data APIs (i.e., Containers),
+format schema, and data I.O. Given a data Container the data translation generates :py:mod:`~hdmf.build.builders`
+for I/O using the relevant :py:mod:`~hdmf.build.objectmapper` to translate attributes from the container
+to definitions in a format schema. Conversely, given a collection of :py:mod:`~hdmf.build.builders`, the
+data translation will construct corresponding frontend :py:mod:`~hdmf.container` using
+:py:mod:`~hdmf.build.objectmapper` to translate fields in the format schema to :py:mod:`~hdmf.container` attributes.
+The :py:mod:`~hdmf.build.manager` then defines functionality to manage the build process to translate containers
+to builders and vice-versa.
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   hdmf.build
+
+Data Format Specification
+=========================
+
+The following modules define data structures and functionality for specifying format schema
+:py:mod:`~hdmf.spec.spec`, managing and reading collections of schema definitions via :py:mod:`~hdmf.spec.namespace`
+and :py:mod:`~hdmf.spec.catalog`, and writing schema via :py:mod:`~hdmf.spec.write`.
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   hdmf.spec
+
+Data API
+========
+
+The modules contained directly at the main hdmf level are used to create user APIs for data standards and
+to enhance interaction with data.
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   hdmf
+   hdmf.container
+   hdmf.utils
+   hdmf.data_utils
+   hdmf.array
+   hdmf.region
+   hdmf.monitor
+   hdmf.query
+   hdmf.region
+
+Testing and Validation Utilities
+================================
+
+Components used to facilitate the definition of unit tests, validation of data files, and validation of format schema.
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   hdmf.validate
+   hdmf.testing
 
 
 :ref:`modindex`

docs/source/conf.py

@@ -49,6 +49,7 @@
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
     "sphinx.ext.napoleon",
     "sphinx.ext.intersphinx",
     "sphinx_gallery.gen_gallery",
@@ -79,6 +80,9 @@
     "zarr": ("https://zarr.readthedocs.io/en/stable/", None),
 }
 
+# Enable sphinx.ext.autosummary to generate automatic summary for pynwb API docs
+autosummary_generate = True
+
 # these links cannot be checked in github actions
 linkcheck_ignore = [
     "https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key",

src/hdmf/__init__.py

@@ -1,3 +1,8 @@
+"""
+The Hierarchical Data Modeling Framework, or HDMF, is a Python package for working with hierarchical data.
+It provides APIs for specifying data models, reading and writing data to different storage backends,
+and representing data with Python objects.
+"""
 from . import query
 from .backends.hdf5.h5_utils import H5Dataset, H5RegionSlicer
 from .container import Container, Data, DataRegion, HERDManager
@@ -12,6 +17,7 @@
     is_method=False,
 )
 def get_region_slicer(**kwargs):
+    """get_region_slicer is deprecated and will be removed in HDMF 4.0"""
     import warnings  # noqa: E402
 
     warnings.warn(

src/hdmf/backends/__init__.py

@@ -1 +1,2 @@
+"""Package for defining new I/O backends for HDMF and using existing backends, e.g., for HDF5. """
 from . import hdf5

src/hdmf/backends/hdf5/__init__.py

@@ -1,3 +1,4 @@
+"""HDF5 backend for reading and writing HDMF data files"""
 from . import h5_utils, h5tools
 from .h5_utils import H5RegionSlicer, H5DataIO
 from .h5tools import HDF5IO, H5SpecWriter, H5SpecReader

src/hdmf/backends/hdf5/h5tools.py

@@ -1,3 +1,4 @@
+"""Module implementing the main HDF5IO HDF5 backend for HDMF"""
 import logging
 import os.path
 import warnings
@@ -35,6 +36,7 @@
 
 
 class HDF5IO(HDMFIO):
+    """HDMF I/O backend for reading and writing HDMF builders to/from HDF5 files."""
 
     __ns_spec_path = 'namespace'  # path to the namespace dataset within a namespace group
 

src/hdmf/backends/io.py

@@ -1,3 +1,4 @@
+"""Module defining the base classes for defining an HDMF I/O backend"""
 from abc import ABCMeta, abstractmethod
 import os
 from pathlib import Path
@@ -10,6 +11,17 @@
 
 
 class HDMFIO(metaclass=ABCMeta):
+    """
+    Base class for implementing a new I/O backend for HDMF
+
+    Derived classes must implement the following abstract methods:
+
+    * :py:meth:`~hdmf.backends.io.HDMFIO.read_builder`
+    * :py:meth:`~hdmf.backends.io.HDMFIO.write_builder`
+    * :py:meth:`~hdmf.backends.io.HDMFIO.open`
+    * :py:meth:`~hdmf.backends.io.HDMFIO.close`
+    * :py:meth:`~hdmf.backends.io.HDMFIO.can_read`
+    """
 
     @staticmethod
     @abstractmethod

src/hdmf/backends/warnings.py

@@ -1,3 +1,6 @@
+"""Collection of warning classes for HDMF I/O"""
+
+
 class BrokenLinkWarning(UserWarning):
     """
     Raised when a group has a key with a None value.

src/hdmf/build/__init__.py

@@ -1,3 +1,4 @@
+"""Package for managing the data translation process between frontend Containers and I/O Builders based on the schema"""
 from .builders import Builder, DatasetBuilder, GroupBuilder, LinkBuilder, ReferenceBuilder, RegionBuilder
 from .classgenerator import CustomClassGenerator, MCIClassGenerator
 from .errors import (BuildError, OrphanContainerBuildError, ReferenceTargetNotBuiltError, ContainerConfigurationError,

src/hdmf/build/builders.py

@@ -1,3 +1,4 @@
+"""Module defining the Builder datastrucures used to represent data for I/O, e.g., GroupBuilder, DatasetBuilder etc."""
 import copy as _copy
 import itertools as _itertools
 import posixpath as _posixpath
@@ -12,6 +13,7 @@
 
 
 class Builder(dict, metaclass=ABCMeta):
+    """Base interface class for all Builder types"""
 
     @docval({'name': 'name', 'type': str, 'doc': 'the name of the group'},
             {'name': 'parent', 'type': 'hdmf.build.builders.Builder', 'doc': 'the parent builder of this Builder',
@@ -75,6 +77,8 @@ def __repr__(self):
 
 
 class BaseBuilder(Builder, metaclass=ABCMeta):
+    """Base class for Builders that support attributes, e.g,. DatasetBuilder and GroupBuilder """
+
     __attribute = 'attributes'  # self dictionary key for attributes
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of the builder.'},

src/hdmf/build/classgenerator.py

@@ -1,3 +1,4 @@
+"""Module iplementing functionaltiy for automatically generating Container classes from type Specs"""
 from copy import deepcopy
 from datetime import datetime, date
 

src/hdmf/build/manager.py

@@ -1,3 +1,4 @@
+"""Collection of classes used to manage the data translation process, e.g., BuildManager and TypeMap. """
 import logging
 from collections import OrderedDict, deque
 from copy import copy

src/hdmf/build/map.py

@@ -1,3 +1,4 @@
+"""DEPRECATED: Classes in map.py should be imported from :py:mod:`~hdmf.build` instead"""
 # this prevents breaking of code that imports these classes directly from map.py
 from .manager import Proxy, BuildManager, TypeSource, TypeMap  # noqa: F401
 from .objectmapper import ObjectMapper  # noqa: F401

src/hdmf/build/objectmapper.py

@@ -1,3 +1,5 @@
+"""Module defining the base ObjectMapper for mapping between Spec objects and AbstractContainer attributes"""
+
 import logging
 import re
 import warnings

src/hdmf/common/__init__.py

@@ -1,5 +1,5 @@
-'''This package will contain functions, classes, and objects
-for reading and writing data in according to the HDMF-common specification
+'''
+Package for reading and writing data in accordance with the HDMF-common-schema specification
 '''
 import os.path
 from copy import deepcopy
@@ -96,6 +96,11 @@ def _dec(cls):
 
 
 def __get_resources():
+    """
+    Get the available schema resources
+
+    :returns: Dict with information about the available namespace YAML file(s)
+    """
     try:
         from importlib.resources import files
     except ImportError:
@@ -112,7 +117,7 @@ def __get_resources():
 
 
 def _get_resources():
-    # LEGACY: Needed to support legacy implementation.
+    """LEGACY: Needed to support legacy implementation, use :py:meth:`~hdmf.common.__get_resources` instead. """
     return __get_resources()
 
 
@@ -129,6 +134,11 @@ def load_namespaces(**kwargs):
 
 
 def available_namespaces():
+    """
+    Get the names of the namespaces available in the NamespaceCatalog in the global __TYPE_MAP.
+
+    :return: Tuple of strings with the names of the available namespaces
+    """
     return __TYPE_MAP.namespace_catalog.namespaces
 
 

src/hdmf/common/io/__init__.py

@@ -1,3 +1,6 @@
+"""
+Object mapper definitions for mapping hdmf-common Containers to Builders and vice-verse
+"""
 from . import multi
 from . import table
 from . import resources

src/hdmf/common/io/alignedtable.py

@@ -1,3 +1,4 @@
+"""Object mapper definitions for types implemented in :py:mod:`~hdmf.common.alignedtable`"""
 from .. import register_map
 from ..alignedtable import AlignedDynamicTable
 from .table import DynamicTableMap

src/hdmf/common/io/multi.py

@@ -1,3 +1,4 @@
+"""Object mapper definitions for types implemented in :py:mod:`~hdmf.common.multi`"""
 from .. import register_map
 from ..multi import SimpleMultiContainer
 from ...build import ObjectMapper