Merge pull request #46 from chrishavlin/ds_caching

dataset cache

docs/_static/yt-napari_0.0.2.json

@@ -0,0 +1,152 @@
+{
+  "title": "InputModel",
+  "type": "object",
+  "properties": {
+    "data": {
+      "title": "Data",
+      "description": "list of datasets to load",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/DataContainer"
+      }
+    }
+  },
+  "definitions": {
+    "ytField": {
+      "title": "ytField",
+      "type": "object",
+      "properties": {
+        "field_type": {
+          "title": "Field Type",
+          "description": "a field type in the yt dataset",
+          "type": "string"
+        },
+        "field_name": {
+          "title": "Field Name",
+          "description": "a field in the yt dataset",
+          "type": "string"
+        },
+        "take_log": {
+          "title": "Take Log",
+          "description": "if true, will apply log10 to the selected data",
+          "default": true,
+          "type": "boolean"
+        }
+      }
+    },
+    "SelectionObject": {
+      "title": "SelectionObject",
+      "type": "object",
+      "properties": {
+        "fields": {
+          "title": "Fields",
+          "description": "list of fields to load for this selection",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ytField"
+          }
+        },
+        "left_edge": {
+          "title": "Left Edge",
+          "description": "the left edge (min x, min y, min z) in units of edge_units",
+          "default": [
+            0.0,
+            0.0,
+            0.0
+          ],
+          "type": "array",
+          "minItems": 3,
+          "maxItems": 3,
+          "items": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "number"
+            },
+            {
+              "type": "number"
+            }
+          ]
+        },
+        "right_edge": {
+          "title": "Right Edge",
+          "description": "the right edge (max x, max y, max z) in units of edge_units",
+          "default": [
+            1.0,
+            1.0,
+            1.0
+          ],
+          "type": "array",
+          "minItems": 3,
+          "maxItems": 3,
+          "items": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "number"
+            },
+            {
+              "type": "number"
+            }
+          ]
+        },
+        "resolution": {
+          "title": "Resolution",
+          "description": "the resolution at which to sample between the edges.",
+          "default": [
+            400,
+            400,
+            400
+          ],
+          "type": "array",
+          "minItems": 3,
+          "maxItems": 3,
+          "items": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "integer"
+            },
+            {
+              "type": "integer"
+            }
+          ]
+        }
+      }
+    },
+    "DataContainer": {
+      "title": "DataContainer",
+      "type": "object",
+      "properties": {
+        "filename": {
+          "title": "Filename",
+          "description": "the filename for the dataset",
+          "type": "string"
+        },
+        "selections": {
+          "title": "Selections",
+          "description": "list of selections to load in this dataset",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/SelectionObject"
+          }
+        },
+        "edge_units": {
+          "title": "Edge Units",
+          "description": "the units to use for left_edge and right_edge in the selections",
+          "default": "code_length",
+          "type": "string"
+        },
+        "store_in_cache": {
+          "title": "Store In Cache",
+          "description": "if enabled, will store references to yt datasets to avoid reloading.",
+          "default": true,
+          "type": "boolean"
+        }
+      }
+    }
+  }
+}

docs/_static/yt-napari_latest.json

@@ -139,6 +139,12 @@
           "description": "the units to use for left_edge and right_edge in the selections",
           "default": "code_length",
           "type": "string"
+        },
+        "store_in_cache": {
+          "title": "Store In Cache",
+          "description": "if enabled, will store references to yt datasets to avoid reloading.",
+          "default": true,
+          "type": "boolean"
         }
       }
     }

docs/quickstart.rst

@@ -7,6 +7,7 @@ After installation, there are three modes of using :code:`yt-napari`:
 2. :ref:`loading a json file from the napari gui<jsonload>`
 3. :ref:`napari gui plugins<naparigui>`
 
+Additionally, you can configure some behavior between napari sessions: see  :ref:`Configuring yt-napari<configfile>`.
 
 .. _jupyusage:
 
@@ -93,3 +94,41 @@ The use the yt Reader plugin, from a Napari viewer, select "Plugins -> yt-napari
 .. image:: _static/readme_ex_003_gui_reader.gif
 
 The reader plugin does its best to align new selections of data with existing yt-napari image layers and should be able to properly align selections from different yt datasets (please submit a bug report if it fails!).
+
+
+.. _configfile:
+
+Configuring yt-napari
+*********************
+
+User options can be saved between napari sessions by adding to the base :code:`yt` configuration
+file, :code:`yt.toml`. :code:`yt` looks for the configuration file in a number of places (check
+out the :code:`yt` documentation
+on:ref:`configuration <https://yt-project.org/doc/reference/configuration.html>`_ ). To add
+:code:`yt-napari` options, open up (or create) the configuration file and add a
+:code:`[yt_napari]` section. An example configuration file might look like:
+
+.. code-block:: bash
+
+    [yt]
+    log_level = 1
+    test_data_dir = "/path/to/yt_data"
+
+    [yt_napari]
+    in_memory_cache = true
+
+
+Configuration options
+#####################
+
+The following options are available:
+
+* :code:`in_memory_cache`, :code:`bool` (default :code:`true`). When :code:`true`,
+the widget and json-readers will store references to yt datasets in an in-memory
+cache. Subsequents loads of the same dataset will then use the available dataset
+handle. This behavior can also be manually controlled in the widget and json
+options -- changing it in the configuration will simply change the default value.
+
+
+Note that boolean values in :code:`toml` files start with lowercase: :code:`true` and
+:code:`false` (instead of :code:`True` and :code:`False`).

docs/schema.rst

@@ -17,4 +17,6 @@ The following versions are available (latest first):
 ..
   schemalistanchor! the following table is auto-generated by repo_utilites/update_schema_docs.py, Do not edit below this line.
 
+yt-napari_0.0.2.json : `view <_static/yt-napari_0.0.2.json>`_ , :download:`download <_static/yt-napari_0.0.2.json>`
+
 yt-napari_0.0.1.json : `view <_static/yt-napari_0.0.1.json>`_ , :download:`download <_static/yt-napari_0.0.1.json>`

repo_utilities/update_schema_docs.py

@@ -1,11 +1,23 @@
 # copies over schema files to the docs/_static folder and updates the schema.rst
+import argparse
+
+from yt_napari._data_model import _store_schema
 from yt_napari.schemas._manager import Manager
 
 
 def run_update(source_dir, schema_dir):
+
     m = Manager(schema_dir)
     m.update_docs(source_dir)
 
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--version", type=str, default=None, help="the schema version to write"
+    )
+
+    args = parser.parse_args()
+    if args.version is not None:
+        _store_schema(version=args.version, overwrite_version=True)
     run_update("./docs", "./src/yt_napari/schemas")

requirements.txt

@@ -8,4 +8,5 @@ sphinx-jsonschema<1.19.0
 Jinja2<3.1.0
 magicgui
 pytest-qt
+platformdirs
 -e .

src/yt_napari/_data_model.py

@@ -4,6 +4,7 @@
 
 from pydantic import BaseModel, Field
 
+from yt_napari.config import ytcfg
 from yt_napari.schemas import _manager
 
 
@@ -42,6 +43,10 @@ class DataContainer(BaseModel):
         "code_length",
         description="the units to use for left_edge and right_edge in the selections",
     )
+    store_in_cache: Optional[bool] = Field(
+        ytcfg.get("yt_napari", "in_memory_cache"),
+        description="if enabled, will store references to yt datasets.",
+    )
 
 
 class InputModel(BaseModel):

src/yt_napari/_ds_cache.py

@@ -0,0 +1,51 @@
+import yt
+
+from yt_napari.config import ytcfg
+from yt_napari.logging import ytnapari_log
+
+
+class DatasetCache:
+    def __init__(self):
+        self.available = {}
+        self._most_recent: str = None
+
+    def add_ds(self, ds, name: str):
+        if name in self.available:
+            ytnapari_log.warning(f"A dataset already exists for {name}. Overwriting.")
+        self.available[name] = ds
+        self._most_recent = name
+
+    @property
+    def most_recent(self):
+        if self._most_recent is not None:
+            return self.available[self._most_recent]
+        return None
+
+    def get_ds(self, name: str):
+        if self.exists(name):
+            return self.available[name]
+        ytnapari_log.warning(f"{name} not found in cache.")
+        return None
+
+    def exists(self, name: str) -> bool:
+        return name in self.available
+
+    def rm_ds(self, name: str):
+        self.available.pop(name, None)
+
+    def rm_all(self):
+        self.available = {}
+        self._most_recent = None
+
+    def check_then_load(self, filename: str):
+        if self.exists(filename):
+            ytnapari_log.info(f"loading {filename} from cache.")
+            return self.get_ds(filename)
+        else:
+            ds = yt.load(filename)
+            if ytcfg.get("yt_napari", "in_memory_cache"):
+                self.add_ds(ds, filename)
+            return ds
+
+
+dataset_cache = DatasetCache()

src/yt_napari/_model_ingestor.py

@@ -1,10 +1,10 @@
 from typing import List, Optional, Tuple, Union
 
 import numpy as np
-import yt
 from unyt import unit_object, unit_registry, unyt_array
 
 from yt_napari._data_model import InputModel
+from yt_napari._ds_cache import dataset_cache
 
 
 def _le_re_to_cen_wid(
@@ -288,7 +288,7 @@ def _process_validated_model(model: InputModel) -> List[SpatialLayer]:
     # dataset and return a plain numpy array
     for m_data in model.data:
 
-        ds = yt.load(m_data.filename)
+        ds = dataset_cache.check_then_load(m_data.filename)
 
         for sel in m_data.selections:
             # get the left, right edge as a unitful array, initialize the layer

src/yt_napari/_tests/test_config.py

@@ -0,0 +1,30 @@
+def test_config_update():
+
+    # note: when the following import is at the top of the file, it errors:
+    # UnboundLocalError: local variable 'ytcfg' referenced before assignment
+    from yt_napari.config import _defaults, _get_updated_config, ytcfg
+
+    current_vals = {}
+    for setting in _defaults.keys():
+        current_vals[setting] = ytcfg.get("yt_napari", setting)
+
+    ytcfg.remove_section("yt_napari")
+    ytcfg = _get_updated_config(ytcfg)
+    assert ytcfg.has_section("yt_napari")
+    for setting, val in _defaults.items():
+        assert val == ytcfg.get("yt_napari", setting)
+
+    # run it through again, make sure existing values are preserved
+    ytcfg.set("yt_napari", "in_memory_cache", False)  # (default is True)
+    ytcfg = _get_updated_config(ytcfg)
+    assert ytcfg.get("yt_napari", "in_memory_cache") is False
+
+    # remove it and add a blank so that the defaults get applied
+    ytcfg.remove_section("yt_napari")
+    ytcfg.add_section("yt_napari")
+    ytcfg = _get_updated_config(ytcfg)
+    for setting, val in _defaults.items():
+        assert val == ytcfg.get("yt_napari", setting)
+
+    # make sure our settings get reset to what they were coming in...
+    ytcfg.update({"yt_napari": current_vals})