From 4d3030ee4908ccdef9333ca87ea7c721c683338e Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Wed, 21 Aug 2024 08:20:36 -0400 Subject: [PATCH 1/6] finished futures/events/notifcations docs --- docs/extending.rst | 4 +- docs/extending/add_devices.rst | 4 +- docs/extending/add_events.rst | 89 ++++++++ docs/extending/add_notifications.rst | 30 +++ docs/index.rst | 2 +- docs/usage.rst | 2 +- docs/usage/backends.rst | 10 + .../micro-manager_backend.rst} | 10 +- docs/usage/data.rst | 9 + docs/usage/devices.rst | 80 +++++-- docs/usage/events.rst | 202 +++++++++++++++++- docs/usage/examples.rst | 10 - docs/usage/futures.rst | 104 ++++++++- docs/usage/installation.rst | 2 +- docs/usage/key_concepts.rst | 3 +- docs/usage/notifications.rst | 33 +-- src/exengine/__init__.py | 1 - .../backends/micromanager/__init__.py | 3 + .../micromanager/mm_device_implementations.py | 12 +- .../micromanager/test/test_mm_camera.py | 2 +- src/exengine/base_classes.py | 6 +- src/exengine/device_types.py | 111 ++++++++++ src/exengine/events/detector_events.py | 19 +- src/exengine/events/multi_d_events.py | 13 +- src/exengine/events/positioner_events.py | 5 +- src/exengine/events/property_events.py | 2 +- .../test/test_multi_d_events_function.py | 16 +- src/exengine/examples/micromanager_example.py | 2 +- src/exengine/examples/using_devices.py | 2 +- .../integration_tests/test_imports.py | 11 +- .../sandbox_test_micromanager_device.py | 2 +- .../work_in_progress/sbox.py | 2 +- src/exengine/kernel/__init__.py | 2 - src/exengine/kernel/device.py | 58 ++++- src/exengine/kernel/device_types_base.py | 173 --------------- src/exengine/kernel/ex_event_base.py | 4 - src/exengine/kernel/test/test_device.py | 3 +- src/exengine/kernel/test/test_executor.py | 6 +- 38 files changed, 733 insertions(+), 316 deletions(-) create mode 100644 docs/extending/add_events.rst create mode 100644 docs/extending/add_notifications.rst create mode 100644 docs/usage/backends.rst rename docs/usage/{examples/micro-manager_backend_example.rst => backends/micro-manager_backend.rst} (91%) create mode 100644 docs/usage/data.rst delete mode 100644 docs/usage/examples.rst create mode 100644 src/exengine/device_types.py delete mode 100644 src/exengine/kernel/device_types_base.py diff --git a/docs/extending.rst b/docs/extending.rst index e341e2d..9a4ff43 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -7,4 +7,6 @@ Extending ExEngine .. toctree:: :maxdepth: 2 - extending/add_devices \ No newline at end of file + extending/add_devices + extending/add_events + extending/add_notifications \ No newline at end of file diff --git a/docs/extending/add_devices.rst b/docs/extending/add_devices.rst index 5fa3e48..fe6631c 100644 --- a/docs/extending/add_devices.rst +++ b/docs/extending/add_devices.rst @@ -13,7 +13,7 @@ All new devices should inherit from the ``Device`` base class or one or more of .. code-block:: python - from exengine.kernel.device_types_base import Device + from exengine.base_classes import Device class ANewDevice(Device): def __init__(self, name): @@ -64,7 +64,7 @@ Specialized device types implement functionality through abstract methods that m .. code-block:: python - from exengine.kernel.device_types_base import Detector + from exengine.device_types import Detector # TODO: may change this API in the future class ANewCameraDevice(Detector): diff --git a/docs/extending/add_events.rst b/docs/extending/add_events.rst new file mode 100644 index 0000000..412d2a7 --- /dev/null +++ b/docs/extending/add_events.rst @@ -0,0 +1,89 @@ +.. _add_events: + +Creating Custom Events +======================= + +Basic Event Creation +-------------------- + +To create a custom event: + +1. Subclass ``ExecutorEvent`` +2. Implement the ``execute()`` method + +.. code-block:: python + + from exengine.base_classes import ExecutorEvent + + class MyCustomEvent(ExecutorEvent): + def execute(self): + # Main event logic goes here + result = self.perform_operation() + return result + + def perform_operation(self): + # Implement your operation here + pass + +Adding Notifications +-------------------- + +To add notifications: + +1. Specify ``notification_types`` +2. Use ``self.publish_notification()`` in ``execute()`` + +.. code-block:: python + + from exengine.notifications import MyCustomNotification + + class MyEventWithNotification(ExecutorEvent): + notification_types = [MyCustomNotification] + + def execute(self): + # Event logic + self.publish_notification(MyCustomNotification(payload="Operation completed")) + +Implementing Capabilities +------------------------- + +Data Producing Capability +^^^^^^^^^^^^^^^^^^^^^^^^^ + +For events that produce data: + +.. code-block:: python + + from exengine.base_classes import ExecutorEvent, DataProducing + + class MyDataProducingEvent(ExecutorEvent, DataProducing): + def execute(self): + data, metadata = self.generate_data() + self.put_data(data_coordinates, data, metadata) + + def generate_data(self): + # Generate your data here + pass + +Stoppable Capability +^^^^^^^^^^^^^^^^^^^^ + +For stoppable events: + +.. code-block:: python + + from exengine.base_classes import ExecutorEvent, Stoppable + + class MyStoppableEvent(ExecutorEvent, Stoppable): + def execute(self): + while not self.is_stop_requested(): + self.do_work() + self.cleanup() + + def do_work(self): + # Implement your work here + pass + + def cleanup(self): + # Cleanup logic here + pass diff --git a/docs/extending/add_notifications.rst b/docs/extending/add_notifications.rst new file mode 100644 index 0000000..63164a5 --- /dev/null +++ b/docs/extending/add_notifications.rst @@ -0,0 +1,30 @@ +.. _add_notifications: + +Creating Custom Notifications +------------------------------ + +To create a custom notification: + +1. Subclass ``exengine.base_classes.Notification`` +2. Use Python's ``@dataclass`` decorator +3. Define ``category`` (from ``exengine.notifications.NotificationCategory`` enum) and ``description`` (string) as class variables +4. Optionally, specify a payload type using a type hint in the class inheritance. For example, ``class MyCustomNotification(Notification[str])`` indicates this notification's payload will be a string. + +Keep payloads lightweight for efficient processing. Example: + +.. code-block:: python + + from dataclasses import dataclass + from exengine.base_classes import Notification + from exengine.notifications import NotificationCategory + + @dataclass + class MyCustomNotification(Notification[str]): + category = NotificationCategory.Device + description = "A custom device status update" + + # Usage + notification = MyCustomNotification(payload="Device XYZ is ready") + + + diff --git a/docs/index.rst b/docs/index.rst index a4d0a57..cf11eaa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ Key Features: 2. **Adaptable to Multiple Frontends**: Compatible with GUIs, scripts, networked automated labs, and AI-integrated microscopy -3. **Advanced Threading Capabilities**: Utilities for parallelization, asynchronous execution, and complex, multi-device workflows. +3. **Powerful Threading Capabilities**: Utilities for parallelization, asynchronous execution, and complex, multi-device workflows. 4. **Modality Agnostic**: Adaptable to diverse microscopy techniques thanks to general purpose design. diff --git a/docs/usage.rst b/docs/usage.rst index 3063266..082fb6e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -9,4 +9,4 @@ Usage usage/installation usage/key_concepts - usage/examples \ No newline at end of file + usage/backends \ No newline at end of file diff --git a/docs/usage/backends.rst b/docs/usage/backends.rst new file mode 100644 index 0000000..c325f0b --- /dev/null +++ b/docs/usage/backends.rst @@ -0,0 +1,10 @@ +.. _backends: + +======== +Backends +======== + +.. toctree:: + :maxdepth: 2 + + backends/micro-manager_backend \ No newline at end of file diff --git a/docs/usage/examples/micro-manager_backend_example.rst b/docs/usage/backends/micro-manager_backend.rst similarity index 91% rename from docs/usage/examples/micro-manager_backend_example.rst rename to docs/usage/backends/micro-manager_backend.rst index 29aae9b..0d5b123 100644 --- a/docs/usage/examples/micro-manager_backend_example.rst +++ b/docs/usage/backends/micro-manager_backend.rst @@ -1,7 +1,7 @@ -.. _micro-manager_backend_example: +.. _micro-manager_backend: ################################################################## -Using ExEngine with Micro-Manager Backend +Micro-Manager Backend ################################################################## Installation @@ -25,11 +25,13 @@ Running the ExEngine .. code-block:: python + # Micro-Manager backend-specific functions from mmpycorex import create_core_instance, download_and_install_mm, terminate_core_instances - from exengine.kernel.executor import ExecutionEngine + + from exengine import ExecutionEngine from exengine.kernel.data_coords import DataCoordinates from exengine.kernel.ex_event_base import DataHandler - from exengine.backends.micromanager.mm_device_implementations import MicroManagerCamera, MicroManagerSingleAxisStage + from exengine.backends.micromanager import MicroManagerCamera, MicroManagerSingleAxisStage from exengine.storage_backends.NDTiffandRAM import NDRAMStorage from exengine.events.detector_events import StartCapture, ReadoutData diff --git a/docs/usage/data.rst b/docs/usage/data.rst new file mode 100644 index 0000000..f19a73e --- /dev/null +++ b/docs/usage/data.rst @@ -0,0 +1,9 @@ +.. _data: + + +======= +Data +======= + + +Data hand \ No newline at end of file diff --git a/docs/usage/devices.rst b/docs/usage/devices.rst index fe3a127..dce1098 100644 --- a/docs/usage/devices.rst +++ b/docs/usage/devices.rst @@ -1,47 +1,81 @@ .. _devices: -####### +======= Devices -####### +======= -Devices in ExEngine are software representations of hardware components in a microscope system. When possible, they provide a consistent way to interact with diverse equipment, abstracting away the complexities of individual hardware implementations. When not possible, devices can additionally expose specialized APIs specific to individual components. -ExEngine supports multiple **backends** - individual devices or libraries of devices (e.g., Micro-Manager). The method to create devices depends on the specific backend in use. +Devices in ExEngine represent hardware or software components. They provide: +- Standardized interfaces for common functionalities +- Thread-safe execution of commands +- Support for multiple backend implementations (i.e. physical hardware devices or libraries for the control of multiple devices) +- Flexibility to represent any grouping of related functionality -Here's a minimal example using the Micro-Manager backend: + +While often used for microscopy hardware, ExEngine's device concept and its benefits are not limited to this domain. A device can represent physical hardware, virtual devices, or software services. + + + +Using Devices +------------- +ExEngine exposes devices through specific backends. + +For example, the Micro-Manager backend enables access to hardware devices controllable through Micro-Manager. (For installation and setup instructions, see :ref:`micro-manager_backend`). + +.. code-block:: python + + # (assuming MM backend already installed and initialized) + # load the micro-manager device for an objective lens switcher + objective = MicroManagerDevice("Objective") + + # Set the objective in use + objective.Label = "Nikon 10X S Fluor" + +Without further specialization, devices are free to have any method and property names. However, certain functionalities are standardized through device types: + + + +Device Types +^^^^^^^^^^^^ +Functionality can be grouped with certain device types: + +For example, the Detector type, which has standardized methods like start, stop, arm, etc. + +Here's a quick example of a Detector: .. code-block:: python - from mmpycorex import create_core_instance - from exengine.kernel.executor import ExecutionEngine - from exengine.backends.micromanager.mm_device_implementations import MicroManagerSingleAxisStage + detector = MicroManagerCamera() + camera.arm(10) # this acquires 10 images + camera.start() - # Create the ExecutionEngine - executor = ExecutionEngine() +Events often take specific device types as parameters. This enables the re-use of events across multiple devices - # Initialize Micro-Manager core - create_core_instance(config_file='MMConfig_demo.cfg') +For example, the ReadoutData event takes a detector: - # Access Micro-Manager device - z_stage = MicroManagerSingleAxisStage() +.. code-block:: python + + readout_event = ReadoutData(detector=camera, ...) - z_stage.set_position(1234.56) -Device Hierarchies -"""""""""""""""""" -Devices in ExEngine exist in hierarchies. All devices must inherit from the exengine.Device base class. Further functionality can be standardized by inheriting from one or more specific device type classes. For example, ``MicroManagerSingleAxisStage`` is a subclass of ``exengine.SingleAxisPositioner``, which is itself a subclass of ``exengine.Device``. +Thread Safety +------------- +By default, all ExEngine devices are made thread-safe. -The advantage of this hierarchical structure is that it standardizes functionality, allowing code to be written for generic device types (e.g., a single axis positioner) that will work with many different device libraries. This approach enables a common API across different libraries of devices, similar to Micro-Manager's device abstraction layer but on a meta-level - spanning multiple device ecosystems rather than just devices within a single project. However, ExEngine's device system is designed to be adaptable. While adhering to the standardized interfaces offers the most benefits, users can still leverage many advantages of the system without implementing these specialized APIs. +This is done under the hood by intercepting and rerouting all device calls to common threads. +This can be turned off by setting the `no_executor` parameter to `True` when initializing a device: + +.. code-block:: python -TODO: thread standardization features of devices (and how to turn off) + device = SomeDevice(no_executor=True) -TODO: calling functions on devices directly -TODO: link to guide to adding backends -TODO: transition to more complex with events +Adding New Device Types +----------------------- +For information on adding new device types, see :ref:`add_devices`. diff --git a/docs/usage/events.rst b/docs/usage/events.rst index 9020861..a8f1545 100644 --- a/docs/usage/events.rst +++ b/docs/usage/events.rst @@ -1,12 +1,204 @@ .. _events: - -####### +====== Events -####### +====== + + +Events in ExEngine are the fundamental units of experimental workflows. They represent discrete tasks or operations that can be submitted to the ExecutionEngine for processing. Events provide a flexible and modular way to construct complex experimental workflows, ranging from simple hardware commands to sophisticated multi-step procedures that may involve data analysis. + +The work of an event is fully contained in its execute method. This method can be called directly to run the event on the current thread: + + +.. code-block:: python + + from exengine.events.positioner_events import SetPosition2DEvent + + # Create an event + move_event = SetPosition2DEvent(device=xy_stage, position=(10.0, 20.0)) + + # Execute the event directly on the current thread + move_event.execute() + +More commonly, events are submitted to the execution engine to be executed asynchronously: + +.. code-block:: python + + from exengine import ExecutionEngine + + engine = ExecutionEngine.get_instance() + + # Submit the event to the execution engine + future = engine.submit(move_event) + # This returns immediately, allowing other operations to continue + + + +The power of this approach lies in its ability to separate the definition of what takes place from the details of how it is executed. While the event defines the operation to be performed, the execution engine manages the scheduling and execution of events across multiple threads. This separation allows for complex workflows to be built up from simple, reusable components, while the execution engine manages the details of scheduling and resource allocation. + +Monitoring Event Progress +-------------------------- + +When an event is submitted to the ExecutionEngine, it is executed on a separate thread. Monitoring its progress is often necessary for several reasons: + +- Program flow control (e.g., blocking until completion) +- User interface updates +- Triggering subsequent actions +- Data retrieval from the event + +ExEngine provides two mechanisms for this: futures and notifications. The :ref:`futures` and :ref:`notifications` provide full details, while a brief overview is given below: + + +Futures +^^^^^^^ +A future is returned when an event is submitted to the ExecutionEngine. This future represents the eventual result of the event and allows for the following: + +- The event's completion can be checked or awaited +- The event's result can be retrieved +- Any data produced by the event can be accessed + +For example: + +.. code-block:: python + + # The event is submitted and a future is obtained + future = engine.submit(event) + + # The event's completion is awaited and its result is obtained + result = future.await_execution() + + +Notifications +^^^^^^^^^^^^^^ + +Notifications offer real-time updates about an event's progress without impeding its execution or that of subsequent events. They are useful for monitoring long-running events or updating user interfaces. They should not be used for resource-intensive operations such as retrieving large amounts of data, as they are intended for lightweight communication. + + +All events emit at minimum an ``EventExecutedNotification`` upon completion. Additional notifications may also be emitted during execution to provide progress updates. + +Available notifications for an event can be checked as follows: + +.. code-block:: python + + print(MyEvent.notification_types) + +A specific notification can be awaited using a future: + +.. code-block:: python + + future = engine.submit(my_event) + future.await_notification(SpecificNotification) + +This approach allows for targeted monitoring of event milestones or state changes. + +Further details can be found in the :ref:`notifications` section. + + +Events that return values +-------------------------- +Some events in ExEngine return values. These values can be retrieved in two ways: + +1. Direct execution: + +When executing an event directly, simply capture the return value: + +.. code-block:: python + + from exengine.events import SomeComputationEvent + + compute_event = SomeComputationEvent(param1=10, param2=20) + result = compute_event.execute() + print(f"Result: {result}") + +2. Asynchronous execution with futures: + +When submitting an event to the ExecutionEngine, use the future to retrieve the result: + +.. code-block:: python + + from exengine import ExecutionEngine + + engine = ExecutionEngine.get_instance() + future = engine.submit(compute_event) + result = future.await_execution() + print(f"Result: {result}") + + + +Composing Complex Workflows +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Events can be combined to create more complex workflows: + +For example, moving an XY stage, capturing an image, and reading out the data and repeating can be expressed as the following sequence of events: + +.. code-block:: python + + from exengine.events import SetPosition2DEvent, StartCapture, ReadoutData, Sleep + + # Create a sequence of events + events = [ + SetPosition2DEvent(device=xy_stage, position=(0, 0)), + StartCapture(detector=camera, num_images=1), + ReadoutData(detector=camera, num_images=1), + Sleep(time_s=1), + SetPosition2DEvent(device=xy_stage, position=(10, 10)), + StartCapture(detector=camera, num_images=1), + ReadoutData(detector=camera, num_images=1), + ] + + # Submit all events + futures = engine.submit(events) + + +.. TODO: compound future or get individual futures + + +Event Capabilities +------------------- + +Events in ExEngine can have special "Capabilities" that extend their functionality. These Capabilities are accessed through methods on the futures returned when submitting events to the ExecutionEngine. + +Data Producing Events +^^^^^^^^^^^^^^^^^^^^^ + +Some events are capable of generating data during their execution. For these events, you can use the ``await_data`` method on the future to retrieve the produced data: + +.. code-block:: python + + future = engine.submit(data_producing_event) + data, metadata = future.await_data(data_coordinates, return_data=True, return_metadata=True) + +This method allows you to wait for specific data to be produced and optionally retrieve both the data and its associated metadata. + +Stoppable Events +^^^^^^^^^^^^^^^^ + +Certain events can be interrupted during their execution. If an event is stoppable, you can use the ``stop`` method on its future: + +.. code-block:: python + + future = engine.submit(stoppable_event) + # ... later ... + future.stop(await_completion=True) + +This method requests the event to stop its execution. The ``await_completion`` parameter determines whether the method should block until the event has stopped. + +Abortable Events +^^^^^^^^^^^^^^^^ + +Similar to stoppable events, abortable events can be terminated, but more abruptly. Use the ``abort`` method on the future: + +.. code-block:: python + + future = engine.submit(abortable_event) + # ... later ... + future.abort(await_completion=True) +This method immediately terminates the event's execution. As with ``stop``, ``await_completion`` determines whether to wait for the abortion to complete. -Events are modular units of instructions and or computation. They can be as simple as a single command like moving the position of a hardware device, or contain multiple steps and computation. They provide building blocks to create more complex experimental workflows. +Creating Custom Events +----------------------- +.. TODO: lambda events -TODO: what else should be said about events? +See the :ref:`add_events` section for more information on creating custom events. \ No newline at end of file diff --git a/docs/usage/examples.rst b/docs/usage/examples.rst deleted file mode 100644 index c870b12..0000000 --- a/docs/usage/examples.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _examples: - -======== -Examples -======== - -.. toctree:: - :maxdepth: 2 - - examples/micro-manager_backend_example \ No newline at end of file diff --git a/docs/usage/futures.rst b/docs/usage/futures.rst index bc05010..4e7e16e 100644 --- a/docs/usage/futures.rst +++ b/docs/usage/futures.rst @@ -1,8 +1,104 @@ .. _futures: - -######## +####### Futures -######## +####### + + +Futures in ExEngine represent the outcome of asynchronous operations. They provide a way to handle long-running tasks without blocking the main execution thread. Futures allow you to submit events for execution and then either wait for their completion or continue with other tasks, checking back later for results. This enables efficient, non-blocking execution of complex workflows. + +When you submit an event to the ExecutionEngine, it returns a future: + +.. code-block:: python + + from exengine import ExecutionEngine + from exengine.events import SomeEvent + + engine = ExecutionEngine.get_instance() + event = SomeEvent() + future = engine.submit(event) + +You can then use this future to interact with the ongoing operation. + +Waiting for Completion + Error Handling +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To wait for an event to complete and get its result: + +.. code-block:: python + + result = future.await_execution() + +This will block until the event completes. + +If an event raises an exception during its execution, you can catch it when awaiting the future: + +.. code-block:: python + + try: + result = future.await_execution() + except Exception as e: + print(f"Event failed with error: {e}") + +Checking Completion Status +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can check if an event has completed without blocking: + +.. code-block:: python + + if future.is_execution_complete(): + print("Event has completed") + else: + print("Event is still running") + + +Notifications +^^^^^^^^^^^^^^ + +Futures can be used to await specific notifications from an event: + +.. code-block:: python + + future.await_notification(SomeSpecificNotification) + +This will block until the specified notification is received. If the notification has already been received when this method is called, it will return immediately. + +Retrieving Data +^^^^^^^^^^^^^^^^^^^^ + +For data-producing events, use the future's ``await_data`` method to retrieve data: + +.. code-block:: python + + # Retrieve a single piece of data + data, metadata = future.await_data({'time': 2}, return_data=True, return_metadata=True) + + # Retrieve multiple pieces of data + data_list = future.await_data([{'time': 0}, {'time': 1}, {'time': 2}], return_data=True) + +The ``data_coordinates`` parameter can specify a single piece of data or multiple pieces. + + +Stopping Execution +^^^^^^^^^^^^^^^^^^^^^ + +If an event is stoppable (inherits from ``Stoppable``), you can use the future to stop it: + +.. code-block:: python + + future.stop(await_completion=True) + +This requests the event to stop its execution. The ``await_completion`` parameter determines whether the method should block until the event has stopped. + +Aborting Execution +^^^^^^^^^^^^^^^^^^^^^ + +Similarly, for abortable events (inheriting from ``Abortable``): + +.. code-block:: python + + future.abort(await_completion=True) + +This immediately terminates the event's execution. As with ``stop``, ``await_completion`` determines whether to wait for the abortion to complete. -Futures represent the outcome of asynchronous operations. They provide a way to handle long-running tasks without blocking the main execution thread. Futures allow you to submit events for execution and then either wait for their completion or continue with other tasks, checking back later for results. This enables efficient, non-blocking execution of complex workflows. diff --git a/docs/usage/installation.rst b/docs/usage/installation.rst index 3ea7a28..78263c8 100644 --- a/docs/usage/installation.rst +++ b/docs/usage/installation.rst @@ -11,7 +11,7 @@ To install ExEngine with all available backends, use pip: pip install "exengine[all]" -The `[all]` option installs ExEngine with all available device and storage backend dependencies. +The ``[all]`` option installs ExEngine with all available device and storage backend dependencies. Alternatively, if you only want to install specific backends, you can use: diff --git a/docs/usage/key_concepts.rst b/docs/usage/key_concepts.rst index c16fa45..c5da42c 100644 --- a/docs/usage/key_concepts.rst +++ b/docs/usage/key_concepts.rst @@ -10,4 +10,5 @@ Key concepts devices events futures - notifications \ No newline at end of file + notifications + data \ No newline at end of file diff --git a/docs/usage/notifications.rst b/docs/usage/notifications.rst index 4ef9944..6eb55b5 100644 --- a/docs/usage/notifications.rst +++ b/docs/usage/notifications.rst @@ -5,9 +5,6 @@ Notifications ============= -Overview ---------- - Notifications in ExEngine provide a powerful mechanism for asynchronous communication between the Execution and user code. They allow devices, events, and other components to broadcast updates about their status or important occurrences. This enables reactive programming patterns, allowing your software to respond dynamically to changes in the system state or experimental conditions. Notifications can serve several purposes: @@ -148,32 +145,4 @@ Events can emit notifications using the ``publish_notification`` method: Creating Custom Notifications ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To create a custom notification: - -1. Subclass ``exengine.base_classes.Notification`` -2. Use Python's ``@dataclass`` decorator -3. Define ``category`` (from ``exengine.notifications.NotificationCategory`` enum) and ``description`` (string) as class variables -4. Optionally, specify a payload type using a type hint in the class inheritance. For example, ``class MyCustomNotification(Notification[str])`` indicates this notification's payload will be a string. - -Keep payloads lightweight for efficient processing. Example: - -.. code-block:: python - - from dataclasses import dataclass - from exengine.base_classes import Notification - from exengine.notifications import NotificationCategory - - @dataclass - class MyCustomNotification(Notification[str]): - category = NotificationCategory.Device - description = "A custom device status update" - - # Usage - notification = MyCustomNotification(payload="Device XYZ is ready") - - - - - - - +See :ref:`add_notifications` . \ No newline at end of file diff --git a/src/exengine/__init__.py b/src/exengine/__init__.py index b4fe1b9..1b3cc6d 100644 --- a/src/exengine/__init__.py +++ b/src/exengine/__init__.py @@ -5,5 +5,4 @@ """ from ._version import __version__, version_info -from . import kernel from .kernel.executor import ExecutionEngine diff --git a/src/exengine/backends/micromanager/__init__.py b/src/exengine/backends/micromanager/__init__.py index e69de29..140b56c 100644 --- a/src/exengine/backends/micromanager/__init__.py +++ b/src/exengine/backends/micromanager/__init__.py @@ -0,0 +1,3 @@ +from .mm_device_implementations import (MicroManagerCamera, MicroManagerDevice, MicroManagerSingleAxisStage, + MicroManagerXYStage) +# TODO: shutter, SLM \ No newline at end of file diff --git a/src/exengine/backends/micromanager/mm_device_implementations.py b/src/exengine/backends/micromanager/mm_device_implementations.py index 0e16cb7..e06c6be 100644 --- a/src/exengine/backends/micromanager/mm_device_implementations.py +++ b/src/exengine/backends/micromanager/mm_device_implementations.py @@ -2,8 +2,8 @@ Implementation of Micro-Manager device_implementations.py in terms of the AcqEng bottom API """ -from exengine.kernel.device_types_base import (Detector, TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) -from exengine.kernel.device_types_base import Device +from device_types import (Detector, TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) +from kernel.device import Device from mmpycorex import Core import numpy as np import pymmcore @@ -228,12 +228,6 @@ def __init__(self, name=None): self._last_snap = None self._snap_active = False - def set_exposure(self, exposure: float) -> None: - self._core_noexec.set_exposure(self._device_name_noexec, exposure) - - def get_exposure(self) -> float: - return self._core_noexec.get_exposure(self._device_name_noexec) - def arm(self, frame_count=None) -> None: if frame_count == 1: # nothing to prepare because snap will be called @@ -268,7 +262,7 @@ def stop(self) -> None: def is_stopped(self) -> bool: return not self._core_noexec.is_sequence_running(self._device_name_noexec) and not self._snap_active - def pop_image(self, timeout=None) -> Tuple[np.ndarray, dict]: + def pop_data(self, timeout=None) -> Tuple[np.ndarray, dict]: if self._frame_count != 1: md = pymmcore.Metadata() start_time = time.time() diff --git a/src/exengine/backends/micromanager/test/test_mm_camera.py b/src/exengine/backends/micromanager/test/test_mm_camera.py index 3f1f541..8db315b 100644 --- a/src/exengine/backends/micromanager/test/test_mm_camera.py +++ b/src/exengine/backends/micromanager/test/test_mm_camera.py @@ -27,7 +27,7 @@ def capture_images(num_images, executor, camera): data_handler = DataHandler(storage=storage) start_capture_event = StartCapture(num_images=num_images, detector=camera) - readout_images_event = ReadoutData(num_images=num_images, detector=camera, + readout_images_event = ReadoutData(number=num_images, detector=camera, data_coordinates_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler) diff --git a/src/exengine/base_classes.py b/src/exengine/base_classes.py index 13e2fbd..9d1da09 100644 --- a/src/exengine/base_classes.py +++ b/src/exengine/base_classes.py @@ -1,2 +1,4 @@ -from .kernel.notification_base import Notification -from .kernel.ex_event_base import ExecutorEvent \ No newline at end of file +from .kernel.notification_base import Notification, NotificationCategory +from .kernel.ex_event_capabilities import DataProducing, Stoppable, Abortable +from .kernel.ex_event_base import ExecutorEvent +from kernel.device import Device \ No newline at end of file diff --git a/src/exengine/device_types.py b/src/exengine/device_types.py new file mode 100644 index 0000000..90486b3 --- /dev/null +++ b/src/exengine/device_types.py @@ -0,0 +1,111 @@ +"""" +Base classes for device_implementations that can be used by the execution engine +""" + +from abc import abstractmethod +from typing import Tuple +import numpy as np +from kernel.device import Device + + +# TODO: could replace hard coded classes with +# T = TypeVar('T') +# Positionaer(Generic[T]): +# and then (float) or (float, float) for number axes +# or make a triggerable mixin that does this + +class SingleAxisPositioner(Device): + """ A positioner that can move along a single axis (e.g. a z drive used as a focus stage) """ + + @abstractmethod + def set_position(self, position: float) -> None: + ... + + @abstractmethod + def get_position(self) -> float: + ... + + +class TriggerableSingleAxisPositioner(SingleAxisPositioner): + """ + A special type of positioner that can accept a sequence of positions to move to when provided external TTL triggers + """ + @abstractmethod + def set_position_sequence(self, positions: np.ndarray) -> None: + ... + + @abstractmethod + def get_triggerable_position_sequence_max_length(self) -> int: + ... + + @abstractmethod + def stop_position_sequence(self) -> None: + ... + + +class DoubleAxisPositioner(Device): + + @abstractmethod + def set_position(self, x: float, y: float) -> None: + """ + Set the position of the positioner. This is a blocking call that will return when the positioner has + reached the desired position + """ + ... + + @abstractmethod + def get_position(self) -> Tuple[float, float]: + ... + +class TriggerableDoubleAxisPositioner(DoubleAxisPositioner): + + @abstractmethod + def set_position_sequence(self, positions: np.ndarray) -> None: + ... + + @abstractmethod + def get_triggerable_position_sequence_max_length(self) -> int: + ... + + @abstractmethod + def stop_position_sequence(self) -> None: + ... + + +class Detector(Device): + """ + Generic class for a camera and the buffer where it stores data + """ + + @abstractmethod + def arm(self, frame_count=None) -> None: + """ + Arms the device before an start command. This optional command validates all the current features for + consistency and prepares the device for a fast start of the Acquisition. If not used explicitly, + this command will be automatically executed at the first AcquisitionStart but will not be repeated + for the subsequent ones unless a feature is changed in the device. + """ + ... + + @abstractmethod + def start(self) -> None: + ... + + # TODO: is it possible to make this return the number of images captured, to know about when to stop readout? + @abstractmethod + def stop(self) -> None: + ... + + @abstractmethod + def is_stopped(self) -> bool: + ... + + @abstractmethod + def pop_data(self, timeout=None) -> Tuple[np.ndarray, dict]: + """ + Get the next image and metadata from the camera buffer. If timeout is None, this function will block until + an image is available. If timeout is a number, this function will block for that many seconds before returning + (None, None) if no image is available + """ + ... + diff --git a/src/exengine/events/detector_events.py b/src/exengine/events/detector_events.py index aedd94b..22c63a7 100644 --- a/src/exengine/events/detector_events.py +++ b/src/exengine/events/detector_events.py @@ -1,13 +1,13 @@ # TODO: may want to abstract this to data-producing device_implementations in general, not just cameras from typing import Iterable, Optional, Union, Dict -from dataclasses import dataclass, field import itertools from exengine.kernel.ex_event_base import ExecutorEvent -from exengine.kernel.device_types_base import Detector +from device_types import Detector from exengine.kernel.data_coords import DataCoordinates, DataCoordinatesIterator from exengine.kernel.notification_base import Notification, NotificationCategory from exengine.kernel.ex_event_capabilities import Stoppable, DataProducing from exengine.kernel.data_handler import DataHandler +from exengine import ExecutionEngine class DataAcquired(Notification[DataCoordinates]): @@ -25,8 +25,8 @@ class ReadoutData(Stoppable, DataProducing, ExecutorEvent): elements (or indefinitely if num_images is None) detector (Union[Detector, str]): The Detector object to read data from. Can be the object itself, or the name of the object in the ExecutionEngine's device registry. - num_images (int): The number of images to read out. If None, the readout will continue until the - image_coordinate_iterator is exhausted or the camera is stopped and no more images are available. + number (int): The number of pieces of data (e.g. images) to read out. If None, the readout will continue until + the data_coordinate_iterator is exhausted or the camera is stopped and no more images are available. stop_on_empty (bool): If True, the readout will stop when the detector is stopped when there is not an image available to read data_handler (DataHandler): The DataHandler object that will handle the data read out by this event @@ -37,24 +37,27 @@ def __init__(self, data_coordinates_iterator: Union[DataCoordinatesIterator, Iterable[DataCoordinates], Iterable[Dict[str, Union[int, str]]]], detector: Optional[Union[Detector, str]] = None, data_handler: DataHandler = None, - num_images: int = None, + number: int = None, stop_on_empty: bool = False): super().__init__(data_coordinates_iterator=data_coordinates_iterator, data_handler=data_handler) self.detector = detector # TODO: why does IDE not like this type hint? - self.num_images = num_images + self.number = number self.stop_on_empty = stop_on_empty def execute(self) -> None: + # if detector is a string, look it up in the device registry + self.detector: Detector = (self.detector if isinstance(self.detector, Detector) + else ExecutionEngine.get_device(self.detector)) # TODO a more efficient way to do this is with callbacks from the camera # but this is not currently implemented, at least for Micro-Manager cameras - image_counter = itertools.count() if self.num_images is None else range(self.num_images) + image_counter = itertools.count() if self.number is None else range(self.number) for image_number, image_coordinates in zip(image_counter, self.data_coordinate_iterator): while True: # check if event.stop has been called if self.is_stop_requested(): return - image, metadata = self.detector.pop_image(timeout=0.01) # only block for 10 ms so stop event can be checked + image, metadata = self.detector.pop_data(timeout=0.01) # only block for 10 ms so stop event can be checked if image is None and self.stop_on_empty: return elif image is not None: diff --git a/src/exengine/events/multi_d_events.py b/src/exengine/events/multi_d_events.py index 9202713..c7f190e 100644 --- a/src/exengine/events/multi_d_events.py +++ b/src/exengine/events/multi_d_events.py @@ -1,13 +1,10 @@ -from exengine.events.positioner_events import SetPosition2DEvent, SetPosition1DEvent +from exengine.events.positioner_events import SetPosition2DEvent from exengine.events.detector_events import StartCapture, ReadoutData -from exengine.kernel.ex_event_base import ExecutorEvent from exengine.events.misc_events import SetTimeEvent -from exengine.kernel.device_types_base import SingleAxisPositioner, DoubleAxisPositioner, Detector +from device_types import SingleAxisPositioner, Detector from exengine.kernel.data_coords import DataCoordinates -from exengine.events.property_events import (SetTriggerablePropertySequencesEvent, - SetPropertiesEvent) +from exengine.events.property_events import (SetPropertiesEvent) from exengine.events.positioner_events import SetTriggerable1DPositionsEvent, SetPosition1DEvent -from exengine.backends.micromanager.mm_utils import read_mm_config_groups from typing import Union, List, Iterable, Optional import numpy as np import copy @@ -168,7 +165,7 @@ def generate_events(event_list, order, coords=None): new_event_list.append(ReadoutData( detector=camera, - num_images=total_sequence_length, + number=total_sequence_length, data_coordinates_iterator=coords_iterator )) @@ -223,7 +220,7 @@ def generate_events(event_list, order, coords=None): # Non-sequenced case: Add StartCapture and ReadoutImages events num_images = 1 event_set.append(StartCapture(detector=camera, num_images=num_images)) - event_set.append(ReadoutData(detector=camera, num_images=num_images, + event_set.append(ReadoutData(detector=camera, number=num_images, data_coordinates_iterator=[DataCoordinates(**coords)])) final_events.append(event_set) diff --git a/src/exengine/events/positioner_events.py b/src/exengine/events/positioner_events.py index 4c567dc..05b2952 100644 --- a/src/exengine/events/positioner_events.py +++ b/src/exengine/events/positioner_events.py @@ -1,10 +1,9 @@ from typing import List, Union, Tuple, Optional, SupportsFloat import numpy as np -from dataclasses import dataclass from exengine.kernel.ex_event_base import ExecutorEvent -from exengine.kernel.device_types_base import (DoubleAxisPositioner, SingleAxisPositioner, - TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) +from device_types import (DoubleAxisPositioner, SingleAxisPositioner, + TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) class SetPosition2DEvent(ExecutorEvent): diff --git a/src/exengine/events/property_events.py b/src/exengine/events/property_events.py index 17fa394..30c7341 100644 --- a/src/exengine/events/property_events.py +++ b/src/exengine/events/property_events.py @@ -1,6 +1,6 @@ from typing import Any, Iterable, Tuple, Union, List from dataclasses import dataclass -from exengine.kernel.device_types_base import Device +from kernel.device import Device from exengine.kernel.executor import ExecutionEngine from exengine.kernel.ex_event_base import ExecutorEvent diff --git a/src/exengine/events/test/test_multi_d_events_function.py b/src/exengine/events/test/test_multi_d_events_function.py index 77dc078..8ee8cd7 100644 --- a/src/exengine/events/test/test_multi_d_events_function.py +++ b/src/exengine/events/test/test_multi_d_events_function.py @@ -32,7 +32,7 @@ def test_sequenced_z_stack(): assert isinstance(events[0], SetTriggerable1DPositionsEvent) assert isinstance(events[1], StartCapture) assert isinstance(events[2], ReadoutData) - assert events[1].num_images == 6 # 6 z-positions + assert events[1].number == 6 # 6 z-positions def test_sequenced_timelapse(): @@ -58,11 +58,11 @@ def test_sequenced_timelapse(): # Check the SetTriggerablePropertySequencesEvent # Check the StartCapture event - assert events[0].num_images == num_time_points + assert events[0].number == num_time_points # Check the ReadoutImages event readout_event = events[1] - assert readout_event.num_images == num_time_points + assert readout_event.number == num_time_points # Check the data coordinate iterator coords = list(readout_event.data_coordinate_iterator) @@ -95,7 +95,7 @@ def test_sequenced_channels(): assert isinstance(events[0], SetTriggerablePropertySequencesEvent) assert isinstance(events[1], StartCapture) assert isinstance(events[2], ReadoutData) - assert events[1].num_images == 3 # 3 channels + assert events[1].number == 3 # 3 channels # TODO: implement channels in multi d @@ -148,7 +148,7 @@ def test_sequenced_channels_and_z_stack_zc_order(): assert isinstance(events[3], ReadoutData) # Check if the number of images is correct (6 z-positions * 2 channels) - assert events[2].num_images == 12 + assert events[2].number == 12 # Check if the number of z-positions is correct assert len(events[0].positions) == 12 @@ -190,7 +190,7 @@ def test_sequenced_channels_and_z_stack_cz_order(): assert isinstance(events[1], SetTriggerable1DPositionsEvent) or isinstance(events[0], SetTriggerable1DPositionsEvent) assert isinstance(events[2], StartCapture) assert isinstance(events[3], ReadoutData) - assert events[2].num_images == 12 + assert events[2].number == 12 expected_channel_sequence = ['DAPI'] * 6 + ['FITC'] * 6 assert events[1].property_sequences[0][2] == expected_channel_sequence @@ -220,7 +220,7 @@ def test_sequenced_time_channels_and_z_stack_tzc_order(): assert isinstance(events[0], SetTriggerablePropertySequencesEvent) or isinstance(events[0], SetTriggerable1DPositionsEvent) assert isinstance(events[1], SetTriggerable1DPositionsEvent) or isinstance(events[1], SetTriggerablePropertySequencesEvent) assert isinstance(events[2], StartCapture) - assert events[2].num_images == 24 # 3 time points * 4 z-positions * 2 channels + assert events[2].number == 24 # 3 time points * 4 z-positions * 2 channels expected_z_sequence = np.array([0, 0, 2, 2, 4, 4, 6, 6, 0, 0, 2, 2, 4, 4, 6, 6, 0, 0, 2, 2, 4, 4, 6, 6]) @@ -257,7 +257,7 @@ def test_sequenced_channels_and_positions(): assert isinstance(events[0], SetTriggerablePropertySequencesEvent) # Channel assert isinstance(events[1], StartCapture) assert isinstance(events[2], ReadoutData) - assert events[1].num_images == 6 # 2 channels * 3 positions + assert events[1].number == 6 # 2 channels * 3 positions expected_channel_sequence = ['DAPI', 'FITC'] * 3 # Repeats for each position assert events[0].property_sequences[0][2] == expected_channel_sequence diff --git a/src/exengine/examples/micromanager_example.py b/src/exengine/examples/micromanager_example.py index 8569f33..faa515c 100644 --- a/src/exengine/examples/micromanager_example.py +++ b/src/exengine/examples/micromanager_example.py @@ -24,7 +24,7 @@ data_handler = DataHandler(storage=NDRAMStorage()) start_capture_event = StartCapture(num_images=num_images, detector=camera) -readout_images_event = ReadoutData(num_images=num_images, detector=camera, +readout_images_event = ReadoutData(number=num_images, detector=camera, data_coordinates_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler) executor.submit(start_capture_event) diff --git a/src/exengine/examples/using_devices.py b/src/exengine/examples/using_devices.py index edd2da7..1a76323 100644 --- a/src/exengine/examples/using_devices.py +++ b/src/exengine/examples/using_devices.py @@ -21,7 +21,7 @@ camera.arm(1) camera.start() -image, metadata = camera.pop_image() +image, metadata = camera.pop_data() print(image) diff --git a/src/exengine/integration_tests/test_imports.py b/src/exengine/integration_tests/test_imports.py index ca41650..88e9e2d 100644 --- a/src/exengine/integration_tests/test_imports.py +++ b/src/exengine/integration_tests/test_imports.py @@ -9,9 +9,11 @@ def test_import_engine(): except ImportError as e: pytest.fail(f"Import failed for ExecutionEngine: {e}") + def test_import_base_classes(): try: - from exengine.base_classes import Notification, ExecutorEvent + from exengine.base_classes import (Notification, ExecutorEvent, NotificationCategory, Abortable, + DataProducing, Stoppable) except ImportError as e: pytest.fail(f"Import failed for base_classes: {e}") @@ -20,3 +22,10 @@ def test_import_notifications(): from exengine.notifications import NotificationCategory, DataStoredNotification, EventExecutedNotification except ImportError as e: pytest.fail(f"Import failed for notifications: {e}") + +def test_mm_imports(): + try: + from exengine.backends.micromanager import (MicroManagerDevice, MicroManagerCamera, + MicroManagerSingleAxisStage, MicroManagerXYStage) + except ImportError as e: + pytest.fail(f"Import failed for MicroManagerDevice: {e}") \ No newline at end of file diff --git a/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py b/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py index 7cc6826..3ee09b2 100644 --- a/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py +++ b/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py @@ -27,7 +27,7 @@ data_handler = DataHandler(storage=storage) start_capture_event = StartCapture(num_images=num_images, camera=camera) -readout_images_event = ReadoutData(num_images=num_images, camera=camera, +readout_images_event = ReadoutData(number=num_images, camera=camera, data_coordinate_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler) executor.submit(start_capture_event) diff --git a/src/exengine/integration_tests/work_in_progress/sbox.py b/src/exengine/integration_tests/work_in_progress/sbox.py index e00952e..01bf45a 100644 --- a/src/exengine/integration_tests/work_in_progress/sbox.py +++ b/src/exengine/integration_tests/work_in_progress/sbox.py @@ -25,7 +25,7 @@ data_handler = DataHandler(storage=NDRAMStorage()) start_capture_event = StartCapture(num_images=num_images, camera=camera) -readout_images_event = ReadoutData(num_images=num_images, camera=camera, +readout_images_event = ReadoutData(number=num_images, camera=camera, image_coordinate_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler) executor.submit(start_capture_event) diff --git a/src/exengine/kernel/__init__.py b/src/exengine/kernel/__init__.py index 76b09d3..e69de29 100644 --- a/src/exengine/kernel/__init__.py +++ b/src/exengine/kernel/__init__.py @@ -1,2 +0,0 @@ -from .notification_base import NotificationCategory, Notification -from .ex_event_base import ExecutorEvent, EventExecutedNotification \ No newline at end of file diff --git a/src/exengine/kernel/device.py b/src/exengine/kernel/device.py index 6488974..735788f 100644 --- a/src/exengine/kernel/device.py +++ b/src/exengine/kernel/device.py @@ -1,9 +1,9 @@ """ Base class for all device_implementations that integrates with the execution engine and enables tokenization of device access. """ -from abc import ABCMeta +from abc import ABCMeta, ABC from functools import wraps -from typing import Any, Dict, Callable +from typing import Any, Dict, Callable, Sequence, Optional, List, Tuple, Iterable, Union from weakref import WeakSet from dataclasses import dataclass @@ -214,3 +214,57 @@ def init_and_register(self, *args, **kwargs): return cls + +class Device(ABC, metaclass=DeviceMetaclass): + """ + Required base class for all devices usable with the execution engine + + Device classes should inherit from this class and implement the abstract methods. The DeviceMetaclass will wrap all + methods and attributes in the class to make them thread-safe and to optionally record all method calls and + attribute accesses. + + Attributes with a trailing _noexec will not be wrapped and will be executed directly on the calling thread. This is + useful for attributes that are not hardware related and can bypass the complexity of the executor. + + Device implementations can also implement functionality through properties (i.e. attributes that are actually + methods) by defining a getter and setter method for the property. + """ + + def __init__(self, name: str, no_executor: bool = False, no_executor_attrs: Sequence[str] = ('_name', )): + """ + Create a new device + + :param name: The name of the device + :param no_executor: If True, all methods and attributes will be executed directly on the calling thread instead + of being rerouted to the executor + :param no_executor_attrs: If no_executor is False, this is a list of attribute names that will be executed + directly on the calling thread + """ + self._no_executor_attrs.extend(no_executor_attrs) + self._no_executor = no_executor + self._name = name + + + def get_allowed_property_values(self, property_name: str) -> Optional[List[str]]: + return None # By default, any value is allowed + + def is_property_read_only(self, property_name: str) -> bool: + return False # By default, properties are writable + + def get_property_limits(self, property_name: str) -> Tuple[Optional[float], Optional[float]]: + return (None, None) # By default, no limits + + def is_property_hardware_triggerable(self, property_name: str) -> bool: + return False # By default, properties are not hardware triggerable + + def get_triggerable_sequence_max_length(self, property_name: str) -> int: + raise NotImplementedError(f"get_triggerable_sequence_max_length is not implemented for {property_name}") + + def load_triggerable_sequence(self, property_name: str, event_sequence: Iterable[Union[str, float, int]]): + raise NotImplementedError(f"load_triggerable_sequence is not implemented for {property_name}") + + def start_triggerable_sequence(self, property_name: str): + raise NotImplementedError(f"start_triggerable_sequence is not implemented for {property_name}") + + def stop_triggerable_sequence(self, property_name: str): + raise NotImplementedError(f"stop_triggerable_sequence is not implemented for {property_name}") diff --git a/src/exengine/kernel/device_types_base.py b/src/exengine/kernel/device_types_base.py deleted file mode 100644 index 31816a1..0000000 --- a/src/exengine/kernel/device_types_base.py +++ /dev/null @@ -1,173 +0,0 @@ -"""" -Base classes for device_implementations that can be used by the execution engine -""" - -from abc import abstractmethod, ABC -from typing import Tuple, List, Iterable, Union, Optional, Sequence -import numpy as np -from .device import DeviceMetaclass - - -class Device(ABC, metaclass=DeviceMetaclass): - """ - Required base class for all devices usable with the execution engine - - Device classes should inherit from this class and implement the abstract methods. The DeviceMetaclass will wrap all - methods and attributes in the class to make them thread-safe and to optionally record all method calls and - attribute accesses. - - Attributes with a trailing _noexec will not be wrapped and will be executed directly on the calling thread. This is - useful for attributes that are not hardware related and can bypass the complexity of the executor. - - Device implementations can also implement functionality through properties (i.e. attributes that are actually - methods) by defining a getter and setter method for the property. - """ - - def __init__(self, name: str, no_executor: bool = False, no_executor_attrs: Sequence[str] = ('_name', )): - """ - Create a new device - - :param name: The name of the device - :param no_executor: If True, all methods and attributes will be executed directly on the calling thread instead - of being rerouted to the executor - :param no_executor_attrs: If no_executor is False, this is a list of attribute names that will be executed - directly on the calling thread - """ - self._no_executor_attrs.extend(no_executor_attrs) - self._no_executor = no_executor - self._name = name - - - def get_allowed_property_values(self, property_name: str) -> Optional[List[str]]: - return None # By default, any value is allowed - - def is_property_read_only(self, property_name: str) -> bool: - return False # By default, properties are writable - - def get_property_limits(self, property_name: str) -> Tuple[Optional[float], Optional[float]]: - return (None, None) # By default, no limits - - def is_property_hardware_triggerable(self, property_name: str) -> bool: - return False # By default, properties are not hardware triggerable - - def get_triggerable_sequence_max_length(self, property_name: str) -> int: - raise NotImplementedError(f"get_triggerable_sequence_max_length is not implemented for {property_name}") - - def load_triggerable_sequence(self, property_name: str, event_sequence: Iterable[Union[str, float, int]]): - raise NotImplementedError(f"load_triggerable_sequence is not implemented for {property_name}") - - def start_triggerable_sequence(self, property_name: str): - raise NotImplementedError(f"start_triggerable_sequence is not implemented for {property_name}") - - def stop_triggerable_sequence(self, property_name: str): - raise NotImplementedError(f"stop_triggerable_sequence is not implemented for {property_name}") - - -# TODO: could replace hard coded classes with -# T = TypeVar('T') -# Positionaer(Generic[T]): -# and then (float) or (float, float) for number axes -# or make a triggerable mixin that does this - -class SingleAxisPositioner(Device): - """ A positioner that can move along a single axis (e.g. a z drive used as a focus stage) """ - - @abstractmethod - def set_position(self, position: float) -> None: - ... - - @abstractmethod - def get_position(self) -> float: - ... - - -class TriggerableSingleAxisPositioner(SingleAxisPositioner): - """ - A special type of positioner that can accept a sequence of positions to move to when provided external TTL triggers - """ - @abstractmethod - def set_position_sequence(self, positions: np.ndarray) -> None: - ... - - @abstractmethod - def get_triggerable_position_sequence_max_length(self) -> int: - ... - - @abstractmethod - def stop_position_sequence(self) -> None: - ... - - -class DoubleAxisPositioner(Device): - - @abstractmethod - def set_position(self, x: float, y: float) -> None: - ... - - @abstractmethod - def get_position(self) -> Tuple[float, float]: - ... - -class TriggerableDoubleAxisPositioner(DoubleAxisPositioner): - - @abstractmethod - def set_position_sequence(self, positions: np.ndarray) -> None: - ... - - @abstractmethod - def get_triggerable_position_sequence_max_length(self) -> int: - ... - - @abstractmethod - def stop_position_sequence(self) -> None: - ... - - -class Detector(Device): - """ - Generic class for a camera and the buffer where it stores data - """ - - # TODO: maybe change these to attributes? - @abstractmethod - def set_exposure(self, exposure: float) -> None: - ... - - @abstractmethod - def get_exposure(self) -> float: - ... - - @abstractmethod - def arm(self, frame_count=None) -> None: - """ - Arms the device before an start command. This optional command validates all the current features for - consistency and prepares the device for a fast start of the Acquisition. If not used explicitly, - this command will be automatically executed at the first AcquisitionStart but will not be repeated - for the subsequent ones unless a feature is changed in the device. - """ - ... - - @abstractmethod - def start(self) -> None: - ... - - # TODO: is it possible to make this return the number of images captured, to know about when to stop readout? - @abstractmethod - def stop(self) -> None: - ... - - @abstractmethod - def is_stopped(self) -> bool: - ... - - # TODO: perhaps this should be a seperate buffer class - # TODO: make this popdata - @abstractmethod - def pop_image(self, timeout=None) -> Tuple[np.ndarray, dict]: - """ - Get the next image and metadata from the camera buffer. If timeout is None, this function will block until - an image is available. If timeout is a number, this function will block for that many seconds before returning - (None, None) if no image is available - """ - ... - diff --git a/src/exengine/kernel/ex_event_base.py b/src/exengine/kernel/ex_event_base.py index 0ce3ef4..49f5114 100644 --- a/src/exengine/kernel/ex_event_base.py +++ b/src/exengine/kernel/ex_event_base.py @@ -69,10 +69,6 @@ def execute(self) -> Any: """ Execute the event. This event is called by the executor, and should be overriden by subclasses to implement the event's functionality. - - Args: - context: Execution context object that holds information related to this specific execution of the event. - (Since the same event can be reused multiple times, this object is unique to each execution of the event.) """ pass diff --git a/src/exengine/kernel/test/test_device.py b/src/exengine/kernel/test/test_device.py index 51b55a5..2615467 100644 --- a/src/exengine/kernel/test/test_device.py +++ b/src/exengine/kernel/test/test_device.py @@ -1,6 +1,7 @@ import pytest from unittest.mock import MagicMock -from exengine.kernel.device_types_base import Device +from kernel.device import Device + @pytest.fixture def mock_device(): diff --git a/src/exengine/kernel/test/test_executor.py b/src/exengine/kernel/test/test_executor.py index 83c9b39..928818a 100644 --- a/src/exengine/kernel/test/test_executor.py +++ b/src/exengine/kernel/test/test_executor.py @@ -4,10 +4,10 @@ """ import pytest -from unittest.mock import MagicMock, create_autospec +from unittest.mock import MagicMock from exengine.kernel.ex_event_base import ExecutorEvent -from exengine.kernel.device_types_base import Device +from kernel.device import Device import time @@ -80,7 +80,7 @@ def test_multiple_method_calls(execution_engine): from concurrent.futures import ThreadPoolExecutor from exengine.kernel.executor import ExecutionEngine -from exengine.kernel.device_types_base import Device +from device_types import Device import threading From 673ff6845d421590242f67d71197838b5184b705 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:09:00 -0400 Subject: [PATCH 2/6] added data documentation --- docs/extending.rst | 3 +- docs/extending/add_devices.rst | 144 ++++++++++++-- docs/extending/add_events.rst | 3 +- docs/extending/add_notifications.rst | 3 +- docs/extending/add_storage.rst | 143 ++++++++++++++ docs/usage/data.rst | 176 +++++++++++++++++- docs/usage/events.rst | 2 + .../micromanager/test/test_mm_camera.py | 2 +- src/exengine/base_classes.py | 3 +- src/exengine/data.py | 5 + src/exengine/examples/micromanager_example.py | 2 +- .../integration_tests/test_imports.py | 8 +- .../_test_multid_events_low_level.py | 2 +- .../sandbox_test_micromanager_device.py | 2 +- .../work_in_progress/sbox.py | 2 +- src/exengine/kernel/data_handler.py | 6 +- ...ta_storage_api.py => data_storage_base.py} | 9 +- src/exengine/kernel/test/test_data_handler.py | 4 +- .../kernel/test/test_notifications.py | 4 +- .../{ => ndtiff_and_ndram}/NDTiffandRAM.py | 4 +- .../{test => ndtiff_and_ndram}/__init__.py | 0 .../test/test_ndtiff_and_ram.py} | 6 +- 22 files changed, 493 insertions(+), 40 deletions(-) create mode 100644 docs/extending/add_storage.rst create mode 100644 src/exengine/data.py rename src/exengine/kernel/{data_storage_api.py => data_storage_base.py} (96%) rename src/exengine/storage_backends/{ => ndtiff_and_ndram}/NDTiffandRAM.py (97%) rename src/exengine/storage_backends/{test => ndtiff_and_ndram}/__init__.py (100%) rename src/exengine/storage_backends/{test/test_NDTiff_and_RAM.py => ndtiff_and_ndram/test/test_ndtiff_and_ram.py} (90%) diff --git a/docs/extending.rst b/docs/extending.rst index 9a4ff43..f224d90 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -9,4 +9,5 @@ Extending ExEngine extending/add_devices extending/add_events - extending/add_notifications \ No newline at end of file + extending/add_notifications + extending/add_storage \ No newline at end of file diff --git a/docs/extending/add_devices.rst b/docs/extending/add_devices.rst index fe6631c..25893e4 100644 --- a/docs/extending/add_devices.rst +++ b/docs/extending/add_devices.rst @@ -4,12 +4,67 @@ Adding Support for New Devices ############################## +We welcome contributions of new device backends to ExEngine! If you've developed a backend for a new device, framework, or system, please consider submitting a Pull Request. + This guide outlines the process of adding support for new devices to the ExEngine framework. -1. Inherit from the Device Base Class -========================================== -All new devices should inherit from the ``Device`` base class or one or more of its subclasses (see `exengine.kernel.device_types_base `_) +Code Organization and packaging +================================ + +When adding a new backend to ExEngine, follow this directory structure: + +.. code-block:: text + + src/exengine/ + └── backends/ + └── your_new_backend/ + ├── __init__.py + ├── your_implementations.py + └── test/ + ├── __init__.py + └── test_your_backend.py + +Replace ``your_new_backend`` with an appropriate name for your backend. + + +Additional dependencies +------------------------ + +If your backend requires additional dependencies, add them to the ``pyproject.toml`` file in the root directory of the project. This ensures that the dependencies are installed when the backend is installed. + +1. Open the ``pyproject.toml`` file in the root directory of the project. +2. Add a new optional dependency group for your backend. For example: + + .. code-block:: toml + + [project.optional-dependencies] + your_new_backend = ["your_dependency1", "your_dependency2"] + +3. Update the ``all`` group to include your new backend: + + .. code-block:: toml + + all = [ + "mmpycorex", + "ndstorage", + + "your_dependency1", + "your_dependency2" + ] + +4. Also add it to the ``device backends`` group, so that it can be installed with ``pip install exengine[your_new_backend]``: + + .. code-block:: toml + + # device backends + your_new_backend = ["dependency1", "dependency2"] + + +Implementing a New Device +=========================== + +All new devices should inherit from the ``Device`` base class or one or more of its subclasses (see `exengine.device_types `_) .. code-block:: python @@ -20,8 +75,8 @@ All new devices should inherit from the ``Device`` base class or one or more of super().__init__(name) # Your device-specific initialization code here -2. Implement Device Functionality -========================================== +Exposing functionality +----------------------- Devices can expose functionality through properties and methods. The base ``Device`` class primarily uses properties. @@ -55,8 +110,8 @@ Here's an example of implementing these special characteristics: # Implement other abstract methods... -3. Use Specialized Device Types -========================================== +Use Specialized Device Types +--------------------------------- There are specialized device types that standardize functionalities through methods. For example, a camera device type will have methods for taking images, setting exposure time, etc. Inheriting from one or more of these devices is recommended whenever possible, as it ensures compatibility with existing workflows and events. @@ -66,18 +121,80 @@ Specialized device types implement functionality through abstract methods that m from exengine.device_types import Detector - # TODO: may change this API in the future class ANewCameraDevice(Detector): - def set_exposure(self, exposure: float) -> None: + def arm(self, frame_count=None): # Implementation here - def get_exposure(self) -> float: + def start(): # Implementation here - # Implement other camera-specific methods... + # Implement other detector-specific methods... + + + +Adding Tests +------------ + +1. Create a ``test_your_backend.py`` file in the ``test/`` directory of your backend. +2. Write pytest test cases for your backend functionality. For example: + + .. code-block:: python + + import pytest + from exengine.backends.your_new_backend import YourNewDevice + + def test_your_device_initialization(): + device = YourNewDevice("TestDevice") + assert device.name == "TestDevice" + + def test_your_device_method(): + device = YourNewDevice("TestDevice") + result = device.some_method() + assert result == expected_value + + # Add more test cases as needed + +Running Tests +------------- + +To run tests for your new backend: + +1. Install the test dependencies. In the ExEngine root directory, run: + + .. code-block:: bash + + pip install -e exengine[test,your_new_backend] + +2. Run pytest for your backend: + + .. code-block:: bash + + pytest -v src/exengine/backends/your_new_backend/test + +Adding documentation +------------------------ + +1. Add documentation for your new backend in the ``docs/`` directory. +2. Create a new RST file, e.g., ``docs/usage/backends/your_new_backend.rst``, describing how to use your backend. +3. Update ``docs/usage/backends.rst`` to include your new backend documentation. + +To build the documentation locally, in the ``exengine/docs`` directory, run: + +.. code-block:: bash + + make clean && make html + +then open ``_build/html/index.html`` in a web browser to view the documentation. + +If you are building the documentation for the first time, you may need to install the required dependencies. In the ``exengine/docs`` directory, run: + +.. code-block:: bash + + pip install -r requirements.txt + Advanced Topics -=============== +----------------- What inheritance from ``Device`` provides ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -125,4 +242,5 @@ Using the first approach allows you to selectively bypass the executor for speci Note that when using ``no_executor_attrs``, you need to specify the names of the attributes or methods as strings in a sequence (e.g., tuple or list) passed to the ``no_executor_attrs`` parameter in the ``super().__init__()`` call. -These approaches provide flexibility in controlling which parts of your device interact with the executor, allowing for optimization where direct access is safe and beneficial. \ No newline at end of file +These approaches provide flexibility in controlling which parts of your device interact with the executor, allowing for optimization where direct access is safe and beneficial. + diff --git a/docs/extending/add_events.rst b/docs/extending/add_events.rst index 412d2a7..f4de5e5 100644 --- a/docs/extending/add_events.rst +++ b/docs/extending/add_events.rst @@ -1,7 +1,8 @@ .. _add_events: +####################### Creating Custom Events -======================= +####################### Basic Event Creation -------------------- diff --git a/docs/extending/add_notifications.rst b/docs/extending/add_notifications.rst index 63164a5..d959041 100644 --- a/docs/extending/add_notifications.rst +++ b/docs/extending/add_notifications.rst @@ -1,7 +1,8 @@ .. _add_notifications: +============================== Creating Custom Notifications ------------------------------- +============================== To create a custom notification: diff --git a/docs/extending/add_storage.rst b/docs/extending/add_storage.rst new file mode 100644 index 0000000..ba531b4 --- /dev/null +++ b/docs/extending/add_storage.rst @@ -0,0 +1,143 @@ +.. _add_storage: + +############################### +Adding New Storage Backends +############################### + +We welcome the addition of new storage backends to ExEngine! If you've created a storage backend that could benefit others, please consider opening a Pull Request. + +This guide outlines the process of creating new data storage backends. + +Code Organization and Packaging +=============================== + +When adding a new storage backend to ExEngine, follow this directory structure: + +.. code-block:: text + + src/exengine/ + └── storage_backends/ + └── your_new_storage/ + ├── __init__.py + ├── your_storage_implementation.py + └── test/ + ├── __init__.py + └── test_your_storage.py + +Replace ``your_new_storage`` with an appropriate name for your storage backend. + +Additional Dependencies +----------------------- + +If your storage backend requires additional dependencies, add them to the ``pyproject.toml`` file: + +1. Open the ``pyproject.toml`` file in the root directory of the project. +2. Add a new optional dependency group for your storage backend: + + .. code-block:: toml + + [project.optional-dependencies] + your_new_storage = ["your_dependency1", "your_dependency2"] + +3. Update the ``all`` group to include your new storage backend: + + .. code-block:: toml + + all = [ + "mmpycorex", + "ndstorage", + "your_dependency1", + "your_dependency2" + ] + +4. Add it to the ``storage backends`` group: + + .. code-block:: toml + + # storage backends + your_new_storage = ["your_dependency1", "your_dependency2"] + +Implementing a New Storage Backend +================================== + +All new storage backends should inherit from the ``DataStorage`` abstract base class. This ensures compatibility with the ExEngine framework. + +The ``DataStorage`` abstract base class is defined in ``exengine/kernel/data_storage_base.py``. You can find the full definition and method requirements there. + +Here's a basic structure for implementing a new storage backend: + +.. code-block:: python + + from exengine.kernel.data_storage_base import DataStorage + + class YourNewStorage(DataStorage): + def __init__(self): + super().__init__() + # Your storage-specific initialization code here + + # Implement the abstract methods from DataStorage + # Refer to data_storage_base.py for the full list of methods to implement + +When implementing your storage backend, make sure to override all abstract methods from the ``DataStorage`` base class. These methods define the interface that ExEngine expects from a storage backend. + +Adding Tests +------------ + +1. Create a ``test_your_storage.py`` file in the ``test/`` directory of your storage backend. +2. Write pytest test cases for your storage backend functionality. For example: + + .. code-block:: python + + import pytest + import numpy as np + from exengine.storage_backends.your_new_storage import YourNewStorage + + def test_your_storage_initialization(): + storage = YourNewStorage() + assert isinstance(storage, YourNewStorage) + + def test_put_and_get_data(): + storage = YourNewStorage() + data = np.array([1, 2, 3]) + metadata = {"key": "value"} + coordinates = {"time": 0, "channel": "DAPI"} + + storage.put(coordinates, data, metadata) + + assert coordinates in storage + np.testing.assert_array_equal(storage.get_data(coordinates), data) + assert storage.get_metadata(coordinates) == metadata + + # Add more test cases as needed + +Running Tests +------------- + +To run tests for your new storage backend: + +1. Install the test dependencies. In the ExEngine root directory, run: + + .. code-block:: bash + + pip install -e ".[test,your_new_storage]" + +2. Run pytest for your storage backend: + + .. code-block:: bash + + pytest -v src/exengine/storage_backends/your_new_storage/test + +Adding Documentation +-------------------- + +1. Add documentation for your new storage backend in the ``docs/`` directory. +2. Create a new RST file, e.g., ``docs/usage/storage_backends/your_new_storage.rst``, describing how to use your storage backend. +3. Update ``docs/usage/storage_backends.rst`` to include your new storage backend documentation. + +To build the documentation locally, in the ``exengine/docs`` directory, run: + +.. code-block:: bash + + make clean && make html + +then open ``_build/html/index.html`` in a web browser to view the documentation. diff --git a/docs/usage/data.rst b/docs/usage/data.rst index f19a73e..b77fd63 100644 --- a/docs/usage/data.rst +++ b/docs/usage/data.rst @@ -5,5 +5,179 @@ Data ======= +ExEngine provides a flexible system for handling acquired data, including mechanisms for identifying, processing, and storing data throughout the experimental workflow. + +DataCoordinates +--------------- + +In microscopy and other multi-dimensional experiments, data often needs to be associated with its position in various dimensions. The ``DataCoordinates`` class is used in ExEngine to represent these positions, providing a flexible way to organize data. + +Concept +^^^^^^^ +``DataCoordinates`` can be thought of as a label for each piece of data in an experiment. For instance, in a time-lapse, multi-channel Z-stack experiment: + +- A full 3D stack might be identified as ``(time=2, channel='GFP')`` +- A single 2D image within that stack could be represented as ``(time=2, channel='GFP', z=10)`` +- A specific pixel in that image might be denoted by ``(time=2, channel='GFP', z=10, x=512, y=512)`` + +Key Features +^^^^^^^^^^^^ +1. **Flexible Axes**: Arbitrary axis names are supported, with convenience methods provided for common ones like 'time', 'channel', and 'z'. +2. **Mixed Value Types**: Coordinate values can be either integers (e.g., for time points) or strings (e.g., for channel names). +3. **Dual Access Methods**: Values can be accessed both as attributes (e.g., coords.time) and as dictionary items (e.g., coords['time']) + +Usage Example +^^^^^^^^^^^^^ + +.. code-block:: python + + from exengine.data import DataCoordinates + + # A DataCoordinates object is created + coords = DataCoordinates(time=3, channel="DAPI", z=5) + + # Values are accessed + print(coords.time) # Output: 3 + print(coords['channel']) # Output: DAPI + + # A new dimension is added + coords.position = 2 + + # It is used as a dictionary + coord_dict = dict(coords) + print(coord_dict) # Output: {'time': 3, 'channel': 'DAPI', 'z': 5, 'position': 2} + + + +.. _data_coordinates_iterator: + +DataCoordinatesIterator +----------------------- + +The ``DataCoordinatesIterator`` class is used to iterate over a sequence of ``DataCoordinates`` objects. It is particularly useful for defining the expected data output of an acquisition event or for iterating over a dataset. + +Key features: + +- Can be created from a ``list`` or ``generator`` of ``DataCoordinates`` objects or valid ``dicts`` +- Both finite and infinite sequences are supported +- Methods are provided to check if specific coordinates might be produced + +Generators can be utilized to create ``DataCoordinatesIterator`` instances efficiently, since the individual ``DataCoordinates`` objects are not created until needed. This essential when the total number of coordinates is unknown or very large. + +Example usage: + +.. code-block:: python + + from exengine.data import DataCoordinatesIterator + import itertools + + # A finite iterator is created from a list of dictionaries + finite_iter = DataCoordinatesIterator.create([ + {'time': 0, 'channel': 'DAPI'}, + {'time': 1, 'channel': 'DAPI'}, + {'time': 2, 'channel': 'DAPI'} + ]) + + # An infinite iterator is created using a generator function + def infinite_coords(): + for i in itertools.count(): + yield {'time': i} + + infinite_iter = DataCoordinatesIterator.create(infinite_coords()) + +DataStorage +------------ + +ExEngine uses a flexible storage API supporting various backends for persistent data storage on disk, in memory, or over networks. + +Like the ``Device`` API, different storage systems are implemented as different backends. Different storage backends can be installed as needed. As described in the :ref:`installation` section, using a particular storage backend requires installing the corresponding module. For example, to save data in the NDTiff format, ExEngine must be installed using ``pip install exengine[all]`` or ``pip install exengine[ndstorage]``. + +Storage backends, like ``Device`` backends, can be installed individually. As outlined in :ref:`installation`, specific backends require their corresponding modules. For instance, NDTiff storage requires installation via ``pip install exengine[ndstorage]``. Alternatively all storage (and device) backends can be installed via ``pip install exengine[all]``. + +For implementing new storage backends, refer to the :ref:`add_storage` section. + + +.. _data_handler: + +DataHandler +----------- + +The ``DataHandler`` acts as a bridge between ``DataProducing`` events and ``DataStorage`` backends in ExEngine. It efficiently manages the flow of data, providing a thread-safe interface for saving, retrieving, and (optionally) processing data. By serving as an intermediary, the ``DataHandler`` ensures efficient data access throughout the experimental pipeline. + +The example below demonstrates how the ``DataHandler`` is initialized with a storage backend and used with ``DataProducing`` ``ExecutorEvents``. It also shows how data can be retrieved, whether from memory or storage, using the ``get`` method. + +.. code-block:: python + + from exengine.data import DataHandler + from exengine.storage import SomeDataStorageImplementation + + # Initialize DataHandler with a storage backend + storage = SomeDataStorageImplementation() + data_handler = DataHandler(storage) + + data_producing_event = SomeDataProducingEvent(data_handler=data_handler) + + # Use with a DataProducing event + engine.submit(data_producing_event) + + # Retrieve data (from memory or storage as needed) + data, metadata = data_handler.get(coords, return_data=True, return_metadata=True) + + + + + + +Data processor +--------------- + +A data processor function allows for optional data processing before storage, useful for tasks like image correction, feature extraction, or data compression. It operates on a separate thread and can be attached to a ``DataHandler``. + +A simple processing function might look like: + +.. code-block:: python + + def process_function(data_coords, data, metadata): + # Modify data or metadata + data[:100, :100] = 0 # Set top-left 100x100 pixels to 0 + metadata['new_key'] = 'new_value' + return data_coords, data, metadata + +The processing function can return: + +1. A tuple of (coordinates, data, metadata) for a single processed image +2. A list of tuples for multiple output images +3. None to discard the data (or to accumulate it for a later call of the processor) + +Example: + +.. code-block:: python + + # Return multiple images + def multi_output_process(coords, data, metadata): + data2 = np.array(data, copy=True) + metadata2 = metadata.copy() + metadata2['Channel'] = 'New_Channel' + return [(coords, data, metadata), (coords, data2, metadata2)] + + # Process multiple images at once + def batch_process(coords, data, metadata): + if not hasattr(batch_process, "images"): + batch_process.images = [] + batch_process.images.append(data) + if len(batch_process.images) == 10: # Process every 10 images + combined = np.stack(batch_process.images, axis=0) + # Process combined data + batch_process.images = [] + return coords, data, metadata + +To use processor functions, attach them to the ``DataHandler``: + +.. code-block:: python + + processor = DataProcessor(process_function) + data_handler = DataHandler(storage, process_fn=processor) + + + -Data hand \ No newline at end of file diff --git a/docs/usage/events.rst b/docs/usage/events.rst index a8f1545..c78cae5 100644 --- a/docs/usage/events.rst +++ b/docs/usage/events.rst @@ -170,6 +170,8 @@ Some events are capable of generating data during their execution. For these eve This method allows you to wait for specific data to be produced and optionally retrieve both the data and its associated metadata. +``DataProducing`` events must have an associated :ref:`DataCoordinatesIterator ` so that the data produced can be uniquely identified, and a :ref:`DataHandler ` so that it knows where to send the data. + Stoppable Events ^^^^^^^^^^^^^^^^ diff --git a/src/exengine/backends/micromanager/test/test_mm_camera.py b/src/exengine/backends/micromanager/test/test_mm_camera.py index 8db315b..664a59a 100644 --- a/src/exengine/backends/micromanager/test/test_mm_camera.py +++ b/src/exengine/backends/micromanager/test/test_mm_camera.py @@ -7,7 +7,7 @@ from exengine.kernel.data_handler import DataHandler from exengine.kernel.data_coords import DataCoordinates from exengine.backends.micromanager.mm_device_implementations import MicroManagerCamera -from exengine.storage_backends.NDTiffandRAM import NDRAMStorage +from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage from exengine.events.detector_events import (StartCapture, ReadoutData, StartContinuousCapture, StopCapture) diff --git a/src/exengine/base_classes.py b/src/exengine/base_classes.py index 9d1da09..2c44948 100644 --- a/src/exengine/base_classes.py +++ b/src/exengine/base_classes.py @@ -1,4 +1,5 @@ from .kernel.notification_base import Notification, NotificationCategory from .kernel.ex_event_capabilities import DataProducing, Stoppable, Abortable from .kernel.ex_event_base import ExecutorEvent -from kernel.device import Device \ No newline at end of file +from .kernel.device import Device +from .kernel.data_storage_base import DataStorage \ No newline at end of file diff --git a/src/exengine/data.py b/src/exengine/data.py new file mode 100644 index 0000000..b9f0b15 --- /dev/null +++ b/src/exengine/data.py @@ -0,0 +1,5 @@ +""" +Convenience file for imports +""" +from .kernel.data_coords import DataCoordinates, DataCoordinatesIterator +from .kernel.data_handler import DataHandler \ No newline at end of file diff --git a/src/exengine/examples/micromanager_example.py b/src/exengine/examples/micromanager_example.py index faa515c..62e71e1 100644 --- a/src/exengine/examples/micromanager_example.py +++ b/src/exengine/examples/micromanager_example.py @@ -3,7 +3,7 @@ from exengine.kernel.data_coords import DataCoordinates from exengine.kernel.ex_event_base import DataHandler from exengine.backends.micromanager.mm_device_implementations import MicroManagerCamera, MicroManagerSingleAxisStage -from exengine.storage_backends.NDTiffandRAM import NDRAMStorage +from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage from exengine.events.detector_events import StartCapture, ReadoutData diff --git a/src/exengine/integration_tests/test_imports.py b/src/exengine/integration_tests/test_imports.py index 88e9e2d..b606de4 100644 --- a/src/exengine/integration_tests/test_imports.py +++ b/src/exengine/integration_tests/test_imports.py @@ -13,7 +13,7 @@ def test_import_engine(): def test_import_base_classes(): try: from exengine.base_classes import (Notification, ExecutorEvent, NotificationCategory, Abortable, - DataProducing, Stoppable) + DataProducing, Stoppable, DataStorage) except ImportError as e: pytest.fail(f"Import failed for base_classes: {e}") @@ -23,6 +23,12 @@ def test_import_notifications(): except ImportError as e: pytest.fail(f"Import failed for notifications: {e}") +def test_import_data(): + try: + from exengine.data import DataCoordinates, DataCoordinatesIterator, DataHandler + except ImportError as e: + pytest.fail(f"Import failed for data: {e}") + def test_mm_imports(): try: from exengine.backends.micromanager import (MicroManagerDevice, MicroManagerCamera, diff --git a/src/exengine/integration_tests/work_in_progress/_test_multid_events_low_level.py b/src/exengine/integration_tests/work_in_progress/_test_multid_events_low_level.py index f6a54c3..4c8cc55 100644 --- a/src/exengine/integration_tests/work_in_progress/_test_multid_events_low_level.py +++ b/src/exengine/integration_tests/work_in_progress/_test_multid_events_low_level.py @@ -12,7 +12,7 @@ ) from exengine.kernel.executor import ExecutionEngine from exengine.kernel.data_handler import DataHandler -from exengine.storage_backends.NDTiffandRAM import NDRAMStorage +from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage from exengine.events.positioner_events import ( SetPosition1DEvent, SetTriggerable1DPositionsEvent, StopTriggerablePositionSequenceEvent ) diff --git a/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py b/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py index 3ee09b2..a632f62 100644 --- a/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py +++ b/src/exengine/integration_tests/work_in_progress/sandbox_test_micromanager_device.py @@ -3,7 +3,7 @@ from exengine.kernel.executor import ExecutionEngine from exengine.kernel.ex_event_base import DataHandler from exengine.backends.micromanager.mm_device_implementations import MicroManagerCamera -from exengine.storage_backends.NDTiffandRAM import NDRAMStorage +from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage from exengine.events.detector_events import StartCapture, ReadoutData from mmpycorex import create_core_instance, terminate_core_instances, get_default_install_location diff --git a/src/exengine/integration_tests/work_in_progress/sbox.py b/src/exengine/integration_tests/work_in_progress/sbox.py index 01bf45a..cb27667 100644 --- a/src/exengine/integration_tests/work_in_progress/sbox.py +++ b/src/exengine/integration_tests/work_in_progress/sbox.py @@ -4,7 +4,7 @@ import os from exengine.kernel.executor import ExecutionEngine from exengine.kernel.ex_event_base import DataHandler -from exengine.storage_backends.NDTiffandRAM import NDRAMStorage +from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage from exengine.events.detector_events import StartCapture, ReadoutData mm_install_dir = get_default_install_location() diff --git a/src/exengine/kernel/data_handler.py b/src/exengine/kernel/data_handler.py index ff830cf..a3aee94 100644 --- a/src/exengine/kernel/data_handler.py +++ b/src/exengine/kernel/data_handler.py @@ -7,7 +7,7 @@ from .notification_base import DataStoredNotification from .data_coords import DataCoordinates -from .data_storage_api import DataStorageAPI +from .data_storage_base import DataStorage from typing import TYPE_CHECKING @@ -49,7 +49,7 @@ class DataHandler: # This class must create at least one additional thread (the saving thread) # and may create another for processing data - def __init__(self, storage: DataStorageAPI, + def __init__(self, storage: DataStorage, process_function: Callable[[DataCoordinates, np.ndarray, JsonValue], Optional[Union[DataCoordinates, np.ndarray, JsonValue, Tuple[DataCoordinates, np.ndarray, JsonValue]]]] = None, @@ -206,7 +206,7 @@ def get(self, coordinates: DataCoordinates, return_data=True, return_metadata=Tr def put(self, coordinates: Any, image: np.ndarray, metadata: Dict, execution_future: Optional["ExecutionFuture"]): """ Hand off this image to the data handler. It will handle handoff to the storage_backends object and image processing - if requested, as well as providing temporary access to the image and metadata as it passes throught this + if requested, as well as providing temporary access to the image and metadata as it passes through this pipeline. If an acquisition future is provided, it will be notified when the image arrives, is processed, and is stored. """ diff --git a/src/exengine/kernel/data_storage_api.py b/src/exengine/kernel/data_storage_base.py similarity index 96% rename from src/exengine/kernel/data_storage_api.py rename to src/exengine/kernel/data_storage_base.py index ece138f..1715a09 100644 --- a/src/exengine/kernel/data_storage_api.py +++ b/src/exengine/kernel/data_storage_base.py @@ -2,17 +2,18 @@ Protocol for storage_backends class that acquisitions ultimate write to where the acquisition data ultimately gets stored """ -from typing import Protocol, runtime_checkable, Union, Dict +from typing import Union, Dict +from abc import ABC from .data_coords import DataCoordinates import numpy as np from pydantic.types import JsonValue -@runtime_checkable -class DataStorageAPI(Protocol): +class DataStorage(ABC): # TODO: about these type hints: better to use the dicts only or also include the DataCoordinates? # DataCoordinates can essentially be used as a dict anyway due to duck typing, so - # maybe its better that other implementations not have to depend on the DataCoordinates class + # is it better to not have to depend on the DataCoordinates class? Perhaps not because storage backends + # are needed anyway def __contains__(self, data_coordinates: Union[DataCoordinates, Dict[str, Union[int, str]]]) -> bool: """Check if item is in the container.""" ... diff --git a/src/exengine/kernel/test/test_data_handler.py b/src/exengine/kernel/test/test_data_handler.py index 28282d1..388df80 100644 --- a/src/exengine/kernel/test/test_data_handler.py +++ b/src/exengine/kernel/test/test_data_handler.py @@ -10,9 +10,9 @@ from exengine.kernel.executor import ExecutionEngine from exengine.kernel.data_handler import DataHandler from exengine.kernel.data_coords import DataCoordinates -from exengine.kernel.data_storage_api import DataStorageAPI +from exengine.kernel.data_storage_base import DataStorage -class MockDataStorage(DataStorageAPI): +class MockDataStorage(DataStorage): def __init__(self): self.data = {} self.metadata = {} diff --git a/src/exengine/kernel/test/test_notifications.py b/src/exengine/kernel/test/test_notifications.py index ea60be3..ebcc12d 100644 --- a/src/exengine/kernel/test/test_notifications.py +++ b/src/exengine/kernel/test/test_notifications.py @@ -7,7 +7,7 @@ from exengine.kernel.notification_base import EventExecutedNotification from exengine.kernel.executor import ExecutionEngine from exengine.kernel.data_handler import DataHandler -from exengine.kernel.data_storage_api import DataStorageAPI +from exengine.kernel.data_storage_base import DataStorage from exengine.kernel.notification_base import DataStoredNotification from exengine.kernel.data_coords import DataCoordinates @@ -27,7 +27,7 @@ def execute(self): @pytest.fixture def mock_storage(): - return Mock(spec=DataStorageAPI) + return Mock(spec=DataStorage) @pytest.fixture def mock_execution_engine(monkeypatch): diff --git a/src/exengine/storage_backends/NDTiffandRAM.py b/src/exengine/storage_backends/ndtiff_and_ndram/NDTiffandRAM.py similarity index 97% rename from src/exengine/storage_backends/NDTiffandRAM.py rename to src/exengine/storage_backends/ndtiff_and_ndram/NDTiffandRAM.py index db16c39..185bc19 100644 --- a/src/exengine/storage_backends/NDTiffandRAM.py +++ b/src/exengine/storage_backends/ndtiff_and_ndram/NDTiffandRAM.py @@ -2,13 +2,13 @@ Adapters for NDTiff and NDRam storage_backends classes """ from typing import Union, Dict -from exengine.kernel.data_storage_api import DataStorageAPI +from exengine.kernel.data_storage_base import DataStorage from exengine.kernel.data_coords import DataCoordinates from ndstorage import NDRAMDataset, NDTiffDataset import numpy as np from pydantic.types import JsonValue -class _NDRAMOrTiffStorage(DataStorageAPI): +class _NDRAMOrTiffStorage(DataStorage): """ Wrapper class for NDTiffDataset and NDRAMDataset to implement the DataStorageAPI protocol """ diff --git a/src/exengine/storage_backends/test/__init__.py b/src/exengine/storage_backends/ndtiff_and_ndram/__init__.py similarity index 100% rename from src/exengine/storage_backends/test/__init__.py rename to src/exengine/storage_backends/ndtiff_and_ndram/__init__.py diff --git a/src/exengine/storage_backends/test/test_NDTiff_and_RAM.py b/src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py similarity index 90% rename from src/exengine/storage_backends/test/test_NDTiff_and_RAM.py rename to src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py index b3a080d..281d93f 100644 --- a/src/exengine/storage_backends/test/test_NDTiff_and_RAM.py +++ b/src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py @@ -1,8 +1,8 @@ import pytest import numpy as np from exengine.kernel.data_coords import DataCoordinates -from exengine.storage_backends.NDTiffandRAM import NDRAMStorage, NDTiffStorage -from exengine.kernel.data_storage_api import DataStorageAPI +from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage, NDTiffStorage +from exengine.kernel.data_storage_base import DataStorage @pytest.fixture(params=["tiff", "ram"]) def data_storage(request, tmp_path): @@ -12,7 +12,7 @@ def data_storage(request, tmp_path): return NDRAMStorage() def test_fully_implements_protocol(data_storage): - assert isinstance(data_storage, DataStorageAPI), "NDStorage does not fully implement DataStorageAPI" + assert isinstance(data_storage, DataStorage), "NDStorage does not fully implement DataStorageAPI" def test_contains_integration(data_storage): data_coordinates = DataCoordinates(coordinate_dict={"time": 1, "channel": "DAPI", "z": 0}) From 07ddfdc49f323f3d94be561f587ab51c0a9b242d Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:43:45 -0400 Subject: [PATCH 3/6] fix imports --- .../backends/micromanager/mm_device_implementations.py | 4 ++-- src/exengine/backends/micromanager/test/test_mm_camera.py | 2 +- src/exengine/device_types.py | 2 +- src/exengine/events/detector_events.py | 2 +- src/exengine/events/multi_d_events.py | 2 +- src/exengine/events/positioner_events.py | 2 +- src/exengine/events/property_events.py | 2 +- src/exengine/kernel/test/test_device.py | 2 +- src/exengine/kernel/test/test_executor.py | 4 ++-- .../ndtiff_and_ndram/test/test_ndtiff_and_ram.py | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/exengine/backends/micromanager/mm_device_implementations.py b/src/exengine/backends/micromanager/mm_device_implementations.py index e06c6be..d716ad1 100644 --- a/src/exengine/backends/micromanager/mm_device_implementations.py +++ b/src/exengine/backends/micromanager/mm_device_implementations.py @@ -2,8 +2,8 @@ Implementation of Micro-Manager device_implementations.py in terms of the AcqEng bottom API """ -from device_types import (Detector, TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) -from kernel.device import Device +from exengine.device_types import (Detector, TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) +from exengine.kernel.device import Device from mmpycorex import Core import numpy as np import pymmcore diff --git a/src/exengine/backends/micromanager/test/test_mm_camera.py b/src/exengine/backends/micromanager/test/test_mm_camera.py index 664a59a..daa03d8 100644 --- a/src/exengine/backends/micromanager/test/test_mm_camera.py +++ b/src/exengine/backends/micromanager/test/test_mm_camera.py @@ -7,7 +7,7 @@ from exengine.kernel.data_handler import DataHandler from exengine.kernel.data_coords import DataCoordinates from exengine.backends.micromanager.mm_device_implementations import MicroManagerCamera -from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage +from exengine.storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage from exengine.events.detector_events import (StartCapture, ReadoutData, StartContinuousCapture, StopCapture) diff --git a/src/exengine/device_types.py b/src/exengine/device_types.py index 90486b3..143ec89 100644 --- a/src/exengine/device_types.py +++ b/src/exengine/device_types.py @@ -5,7 +5,7 @@ from abc import abstractmethod from typing import Tuple import numpy as np -from kernel.device import Device +from .kernel.device import Device # TODO: could replace hard coded classes with diff --git a/src/exengine/events/detector_events.py b/src/exengine/events/detector_events.py index 22c63a7..9e47cc4 100644 --- a/src/exengine/events/detector_events.py +++ b/src/exengine/events/detector_events.py @@ -2,7 +2,7 @@ from typing import Iterable, Optional, Union, Dict import itertools from exengine.kernel.ex_event_base import ExecutorEvent -from device_types import Detector +from exengine.device_types import Detector from exengine.kernel.data_coords import DataCoordinates, DataCoordinatesIterator from exengine.kernel.notification_base import Notification, NotificationCategory from exengine.kernel.ex_event_capabilities import Stoppable, DataProducing diff --git a/src/exengine/events/multi_d_events.py b/src/exengine/events/multi_d_events.py index c7f190e..17fbbcb 100644 --- a/src/exengine/events/multi_d_events.py +++ b/src/exengine/events/multi_d_events.py @@ -1,7 +1,7 @@ from exengine.events.positioner_events import SetPosition2DEvent from exengine.events.detector_events import StartCapture, ReadoutData from exengine.events.misc_events import SetTimeEvent -from device_types import SingleAxisPositioner, Detector +from exengine.device_types import SingleAxisPositioner, Detector from exengine.kernel.data_coords import DataCoordinates from exengine.events.property_events import (SetPropertiesEvent) from exengine.events.positioner_events import SetTriggerable1DPositionsEvent, SetPosition1DEvent diff --git a/src/exengine/events/positioner_events.py b/src/exengine/events/positioner_events.py index 05b2952..28607f9 100644 --- a/src/exengine/events/positioner_events.py +++ b/src/exengine/events/positioner_events.py @@ -2,7 +2,7 @@ import numpy as np from exengine.kernel.ex_event_base import ExecutorEvent -from device_types import (DoubleAxisPositioner, SingleAxisPositioner, +from exengine.device_types import (DoubleAxisPositioner, SingleAxisPositioner, TriggerableSingleAxisPositioner, TriggerableDoubleAxisPositioner) diff --git a/src/exengine/events/property_events.py b/src/exengine/events/property_events.py index 30c7341..eb5cb1b 100644 --- a/src/exengine/events/property_events.py +++ b/src/exengine/events/property_events.py @@ -1,6 +1,6 @@ from typing import Any, Iterable, Tuple, Union, List from dataclasses import dataclass -from kernel.device import Device +from exengine.kernel.device import Device from exengine.kernel.executor import ExecutionEngine from exengine.kernel.ex_event_base import ExecutorEvent diff --git a/src/exengine/kernel/test/test_device.py b/src/exengine/kernel/test/test_device.py index 2615467..9614581 100644 --- a/src/exengine/kernel/test/test_device.py +++ b/src/exengine/kernel/test/test_device.py @@ -1,6 +1,6 @@ import pytest from unittest.mock import MagicMock -from kernel.device import Device +from exengine.kernel.device import Device @pytest.fixture diff --git a/src/exengine/kernel/test/test_executor.py b/src/exengine/kernel/test/test_executor.py index 928818a..cb46e48 100644 --- a/src/exengine/kernel/test/test_executor.py +++ b/src/exengine/kernel/test/test_executor.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock from exengine.kernel.ex_event_base import ExecutorEvent -from kernel.device import Device +from exengine.kernel.device import Device import time @@ -80,7 +80,7 @@ def test_multiple_method_calls(execution_engine): from concurrent.futures import ThreadPoolExecutor from exengine.kernel.executor import ExecutionEngine -from device_types import Device +from exengine.device_types import Device import threading diff --git a/src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py b/src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py index 281d93f..fdc9c55 100644 --- a/src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py +++ b/src/exengine/storage_backends/ndtiff_and_ndram/test/test_ndtiff_and_ram.py @@ -1,7 +1,7 @@ import pytest import numpy as np from exengine.kernel.data_coords import DataCoordinates -from storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage, NDTiffStorage +from exengine.storage_backends.ndtiff_and_ndram.NDTiffandRAM import NDRAMStorage, NDTiffStorage from exengine.kernel.data_storage_base import DataStorage @pytest.fixture(params=["tiff", "ram"]) From b7b9b21b96f76fabfbaa432a4d7b0ea5321a8de1 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:17:35 -0400 Subject: [PATCH 4/6] fix terminology to fix tests --- src/exengine/events/detector_events.py | 37 +++++++++++-------- src/exengine/events/multi_d_events.py | 8 ++-- .../test/test_multi_d_events_function.py | 16 ++++---- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/exengine/events/detector_events.py b/src/exengine/events/detector_events.py index 9e47cc4..04e4a2e 100644 --- a/src/exengine/events/detector_events.py +++ b/src/exengine/events/detector_events.py @@ -17,7 +17,7 @@ class DataAcquired(Notification[DataCoordinates]): class ReadoutData(Stoppable, DataProducing, ExecutorEvent): """ - Readout one or more pieces of data (e.g. images) and associated metadata from a Detector device (e.g. a camera) + Readout one or more blocks of data (e.g. images) and associated metadata from a Detector device (e.g. a camera) Args: data_coordinate_iterator (Iterable[DataCoordinates]): An iterator or list of DataCoordinates objects, which @@ -25,10 +25,10 @@ class ReadoutData(Stoppable, DataProducing, ExecutorEvent): elements (or indefinitely if num_images is None) detector (Union[Detector, str]): The Detector object to read data from. Can be the object itself, or the name of the object in the ExecutionEngine's device registry. - number (int): The number of pieces of data (e.g. images) to read out. If None, the readout will continue until - the data_coordinate_iterator is exhausted or the camera is stopped and no more images are available. - stop_on_empty (bool): If True, the readout will stop when the detector is stopped when there is not an - image available to read + num_blocks (int): The number of pieces of data (e.g. images) to read out. If None, the readout will continue until + the data_coordinate_iterator is exhausted or the Detector is stopped and no more images are available. + stop_on_empty (bool): If True, the readout will stop when the detector is stopped when there is no data + available to read data_handler (DataHandler): The DataHandler object that will handle the data read out by this event """ notification_types = [DataAcquired] @@ -37,11 +37,11 @@ def __init__(self, data_coordinates_iterator: Union[DataCoordinatesIterator, Iterable[DataCoordinates], Iterable[Dict[str, Union[int, str]]]], detector: Optional[Union[Detector, str]] = None, data_handler: DataHandler = None, - number: int = None, + num_blocks: int = None, stop_on_empty: bool = False): super().__init__(data_coordinates_iterator=data_coordinates_iterator, data_handler=data_handler) self.detector = detector # TODO: why does IDE not like this type hint? - self.number = number + self.num_blocks = num_blocks self.stop_on_empty = stop_on_empty @@ -51,7 +51,7 @@ def execute(self) -> None: else ExecutionEngine.get_device(self.detector)) # TODO a more efficient way to do this is with callbacks from the camera # but this is not currently implemented, at least for Micro-Manager cameras - image_counter = itertools.count() if self.number is None else range(self.number) + image_counter = itertools.count() if self.num_blocks is None else range(self.num_blocks) for image_number, image_coordinates in zip(image_counter, self.data_coordinate_iterator): while True: # check if event.stop has been called @@ -69,20 +69,27 @@ def execute(self) -> None: class StartCapture(ExecutorEvent): """ - Special device instruction that captures images from a camera + Special device instruction that captures images from a Detector device (e.g. a camera) """ - def __init__(self, num_images: int, detector: Optional[Detector] = None): + def __init__(self, num_blocks: int, detector: Optional[Detector] = None): + """ + Args: + num_blocks (int): The of pieces of data to capture (i.e. images on a camera) + detector (Union[Detector, str]): The Detector object to capture images from. Can be the object itself, + or the name of the object in the ExecutionEngine's device registry. If None, it will be inferred at + runtime + """ super().__init__() - self.num_images = num_images + self.num_blocks = num_blocks self.detector = detector def execute(self): """ - Capture images from the camera + Capture images from the detector """ try: - self.detector.arm(self.num_images) + self.detector.arm(self.num_blocks) self.detector.start() except Exception as e: self.detector.stop() @@ -90,7 +97,7 @@ def execute(self): class StartContinuousCapture(ExecutorEvent): """ - Tell data-producing device to start capturing images continuously, until a stop signal is received + Tell Detector device to start capturing images continuously, until a stop signal is received """ def __init__(self, camera: Optional[Detector] = None): @@ -110,7 +117,7 @@ def execute(self): class StopCapture(ExecutorEvent): """ - Tell data-producing device to start capturing images continuously, until a stop signal is received + Tell Detector device to start capturing data continuously, until a stop signal is received """ def __init__(self, camera: Optional[Detector] = None): diff --git a/src/exengine/events/multi_d_events.py b/src/exengine/events/multi_d_events.py index 17fbbcb..e4a45fb 100644 --- a/src/exengine/events/multi_d_events.py +++ b/src/exengine/events/multi_d_events.py @@ -149,7 +149,7 @@ def generate_events(event_list, order, coords=None): # )) # Add StartCapture event - new_event_list.append(StartCapture(detector=camera, num_images=total_sequence_length)) + new_event_list.append(StartCapture(detector=camera, num_blocks=total_sequence_length)) # Create data coordinates for ReadoutImages axes_names = {"t": "time", "z": "z", "c": "channel", "p": "position"} @@ -165,7 +165,7 @@ def generate_events(event_list, order, coords=None): new_event_list.append(ReadoutData( detector=camera, - number=total_sequence_length, + num_blocks=total_sequence_length, data_coordinates_iterator=coords_iterator )) @@ -219,8 +219,8 @@ def generate_events(event_list, order, coords=None): if sequence is None: # Non-sequenced case: Add StartCapture and ReadoutImages events num_images = 1 - event_set.append(StartCapture(detector=camera, num_images=num_images)) - event_set.append(ReadoutData(detector=camera, number=num_images, + event_set.append(StartCapture(detector=camera, num_blocks=num_images)) + event_set.append(ReadoutData(detector=camera, num_blocks=num_images, data_coordinates_iterator=[DataCoordinates(**coords)])) final_events.append(event_set) diff --git a/src/exengine/events/test/test_multi_d_events_function.py b/src/exengine/events/test/test_multi_d_events_function.py index 8ee8cd7..c16897b 100644 --- a/src/exengine/events/test/test_multi_d_events_function.py +++ b/src/exengine/events/test/test_multi_d_events_function.py @@ -32,7 +32,7 @@ def test_sequenced_z_stack(): assert isinstance(events[0], SetTriggerable1DPositionsEvent) assert isinstance(events[1], StartCapture) assert isinstance(events[2], ReadoutData) - assert events[1].number == 6 # 6 z-positions + assert events[1].num_blocks == 6 # 6 z-positions def test_sequenced_timelapse(): @@ -58,11 +58,11 @@ def test_sequenced_timelapse(): # Check the SetTriggerablePropertySequencesEvent # Check the StartCapture event - assert events[0].number == num_time_points + assert events[0].num_blocks == num_time_points # Check the ReadoutImages event readout_event = events[1] - assert readout_event.number == num_time_points + assert readout_event.num_blocks == num_time_points # Check the data coordinate iterator coords = list(readout_event.data_coordinate_iterator) @@ -95,7 +95,7 @@ def test_sequenced_channels(): assert isinstance(events[0], SetTriggerablePropertySequencesEvent) assert isinstance(events[1], StartCapture) assert isinstance(events[2], ReadoutData) - assert events[1].number == 3 # 3 channels + assert events[1].num_blocks == 3 # 3 channels # TODO: implement channels in multi d @@ -148,7 +148,7 @@ def test_sequenced_channels_and_z_stack_zc_order(): assert isinstance(events[3], ReadoutData) # Check if the number of images is correct (6 z-positions * 2 channels) - assert events[2].number == 12 + assert events[2].num_blocks == 12 # Check if the number of z-positions is correct assert len(events[0].positions) == 12 @@ -190,7 +190,7 @@ def test_sequenced_channels_and_z_stack_cz_order(): assert isinstance(events[1], SetTriggerable1DPositionsEvent) or isinstance(events[0], SetTriggerable1DPositionsEvent) assert isinstance(events[2], StartCapture) assert isinstance(events[3], ReadoutData) - assert events[2].number == 12 + assert events[2].num_blocks == 12 expected_channel_sequence = ['DAPI'] * 6 + ['FITC'] * 6 assert events[1].property_sequences[0][2] == expected_channel_sequence @@ -220,7 +220,7 @@ def test_sequenced_time_channels_and_z_stack_tzc_order(): assert isinstance(events[0], SetTriggerablePropertySequencesEvent) or isinstance(events[0], SetTriggerable1DPositionsEvent) assert isinstance(events[1], SetTriggerable1DPositionsEvent) or isinstance(events[1], SetTriggerablePropertySequencesEvent) assert isinstance(events[2], StartCapture) - assert events[2].number == 24 # 3 time points * 4 z-positions * 2 channels + assert events[2].num_blocks == 24 # 3 time points * 4 z-positions * 2 channels expected_z_sequence = np.array([0, 0, 2, 2, 4, 4, 6, 6, 0, 0, 2, 2, 4, 4, 6, 6, 0, 0, 2, 2, 4, 4, 6, 6]) @@ -257,7 +257,7 @@ def test_sequenced_channels_and_positions(): assert isinstance(events[0], SetTriggerablePropertySequencesEvent) # Channel assert isinstance(events[1], StartCapture) assert isinstance(events[2], ReadoutData) - assert events[1].number == 6 # 2 channels * 3 positions + assert events[1].num_blocks == 6 # 2 channels * 3 positions expected_channel_sequence = ['DAPI', 'FITC'] * 3 # Repeats for each position assert events[0].property_sequences[0][2] == expected_channel_sequence From d9bc9989ce54f3a32cbe32e05a2d7f464fe07311 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:03:31 -0400 Subject: [PATCH 5/6] add documentation for storage backend and other improvements --- docs/extending/add_devices.rst | 13 ++-- docs/extending/add_storage.rst | 15 +++- docs/usage.rst | 4 +- docs/usage/backends.rst | 22 ++++-- docs/usage/backends/micro-manager_backend.rst | 71 +++++++++++++++---- .../usage/backends/ndtiff_and_ram_backend.rst | 51 +++++++++++++ docs/usage/installation.rst | 56 +++------------ src/exengine/examples/micromanager_example.py | 4 +- .../integration_tests/test_imports.py | 7 ++ .../ndtiff_and_ndram/__init__.py | 1 + 10 files changed, 166 insertions(+), 78 deletions(-) create mode 100644 docs/usage/backends/ndtiff_and_ram_backend.rst diff --git a/docs/extending/add_devices.rst b/docs/extending/add_devices.rst index 25893e4..339e08b 100644 --- a/docs/extending/add_devices.rst +++ b/docs/extending/add_devices.rst @@ -27,6 +27,7 @@ When adding a new backend to ExEngine, follow this directory structure: Replace ``your_new_backend`` with an appropriate name for your backend. +You may also want to edit the ``__init__.py`` file in the ``your_new_backend`` directory to import the Classes for each device you implement in the ``your_implementations.py`` or other files (see the micro-manager backend for an example of this). Additional dependencies ------------------------ @@ -178,7 +179,13 @@ Adding documentation 2. Create a new RST file, e.g., ``docs/usage/backends/your_new_backend.rst``, describing how to use your backend. 3. Update ``docs/usage/backends.rst`` to include your new backend documentation. -To build the documentation locally, in the ``exengine/docs`` directory, run: +To build the documentation locally, you may need to install the required dependencies. In the ``exengine/docs`` directory, run: + +.. code-block:: bash + + pip install -r requirements.txt + +Then, to build, in the ``exengine/docs`` directory, run: .. code-block:: bash @@ -186,11 +193,7 @@ To build the documentation locally, in the ``exengine/docs`` directory, run: then open ``_build/html/index.html`` in a web browser to view the documentation. -If you are building the documentation for the first time, you may need to install the required dependencies. In the ``exengine/docs`` directory, run: - -.. code-block:: bash - pip install -r requirements.txt Advanced Topics diff --git a/docs/extending/add_storage.rst b/docs/extending/add_storage.rst index ba531b4..16b122a 100644 --- a/docs/extending/add_storage.rst +++ b/docs/extending/add_storage.rst @@ -26,6 +26,8 @@ When adding a new storage backend to ExEngine, follow this directory structure: Replace ``your_new_storage`` with an appropriate name for your storage backend. +You may also want to edit the ``__init__.py`` file in the ``your_new_storage`` directory to import the Class from your storage implementation file (see the NDStorage backend for an example of this). + Additional Dependencies ----------------------- @@ -131,13 +133,20 @@ Adding Documentation -------------------- 1. Add documentation for your new storage backend in the ``docs/`` directory. -2. Create a new RST file, e.g., ``docs/usage/storage_backends/your_new_storage.rst``, describing how to use your storage backend. -3. Update ``docs/usage/storage_backends.rst`` to include your new storage backend documentation. +2. Create a new RST file, e.g., ``docs/usage/backends/your_new_storage.rst``, describing how to use your storage backend. +3. Update ``docs/usage/backends.rst`` to include your new storage backend documentation. + +To build the documentation locally, you may need to install the required dependencies. In the ``exengine/docs`` directory, run: + +.. code-block:: bash + + pip install -r requirements.txt -To build the documentation locally, in the ``exengine/docs`` directory, run: +Then, to build, in the ``exengine/docs`` directory, run: .. code-block:: bash make clean && make html then open ``_build/html/index.html`` in a web browser to view the documentation. + diff --git a/docs/usage.rst b/docs/usage.rst index 082fb6e..0144b6b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -8,5 +8,5 @@ Usage :maxdepth: 2 usage/installation - usage/key_concepts - usage/backends \ No newline at end of file + usage/backends + usage/key_concepts \ No newline at end of file diff --git a/docs/usage/backends.rst b/docs/usage/backends.rst index c325f0b..b3a7f72 100644 --- a/docs/usage/backends.rst +++ b/docs/usage/backends.rst @@ -1,10 +1,22 @@ .. _backends: -======== -Backends -======== +================ +Device Backends +================ + +This page contains a list of supported backends for ExEngine, as well as information and setup instructions for each + +.. toctree:: + :maxdepth: 1 + + backends/micro-manager_backend + + +================ +Storage Backends +================ .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - backends/micro-manager_backend \ No newline at end of file + backends/ndstorage_backend \ No newline at end of file diff --git a/docs/usage/backends/micro-manager_backend.rst b/docs/usage/backends/micro-manager_backend.rst index 0d5b123..0bf3c48 100644 --- a/docs/usage/backends/micro-manager_backend.rst +++ b/docs/usage/backends/micro-manager_backend.rst @@ -4,24 +4,70 @@ Micro-Manager Backend ################################################################## +The Micro-Manager Device backend provides access to Micro-Manager devices, such as cameras and stages, +for use with the ExEngine. + Installation ------------ -1. Install ExEngine, including the `micromanager` and `ndstorage` backends: +1. Install ExEngine, including the `micromanager` backends: + + .. code-block:: bash + + pip install "exengine[micromanager]" + + +2. Install Micro-Manager, either by downloding a nightly build from the `Micro-Manager website `_, or through python by typing: + + .. code-block:: python + + from mmpycorex import download_and_install_mm + download_and_install_mm() + + +3. Configure your devices: + + After installation, you need to open Micro-Manager and create a configuration file for your devices. This process involves setting up and saving the hardware configuration for your specific microscope setup. + + For detailed instructions on creating a Micro-Manager configuration file, please refer to the `Micro-Manager Configuration Guide `_. + +4. Launch Micro-Manager pointing to the instance you installed and load the config file you made: + + .. code-block:: python + + from mmpycorex import create_core_instance - .. code-block:: bash + create_core_instance(mm_app_path='/path/to/micro-manager', mm_config_path='name_of_config.cfg') - pip install "exengine[micromanager, ndstorage]" + For testing purposes, if you call ``create_core_instance()`` with no arguments it will default to the default installation path of ``download_and_install_mm()`` and the Micro-Manager demo configuration file. -2. Install Micro-Manager: + +5. Verify the setup: .. code-block:: python - from mmpycorex import download_and_install_mm - download_and_install_mm() + from mmpycorex import Core + + core = Core() + + print(core.get_loaded_devices()) + + This should print a list of all devices loaded from your configuration file. + + + + +Running ExEngine with Micro-Manager +----------------------------------- + +This section shows how to use ExEngine with the Micro-Manager backend. Data is stored in RAM using, +the NDstorage, the backend for which must be installed with: + +.. code-block:: bash + + pip install "exengine[ndstorage]" -Running the ExEngine --------------------- +First, start by launching Micro-Manager and getting access to the loaded devices .. code-block:: python @@ -29,10 +75,9 @@ Running the ExEngine from mmpycorex import create_core_instance, download_and_install_mm, terminate_core_instances from exengine import ExecutionEngine - from exengine.kernel.data_coords import DataCoordinates - from exengine.kernel.ex_event_base import DataHandler + from exengine.data import DataCoordinates, DataHandler from exengine.backends.micromanager import MicroManagerCamera, MicroManagerSingleAxisStage - from exengine.storage_backends.NDTiffandRAM import NDRAMStorage + from exengine.storage_backends.ndtiff_and_ndram import NDRAMStorage from exengine.events.detector_events import StartCapture, ReadoutData # Start Micro-Manager core instance with Demo config @@ -51,8 +96,8 @@ Example 1: Use the ExEngine to Acquire a Timelapse # Capture 100 images on the camera num_images = 100 data_handler = DataHandler(storage=NDRAMStorage()) - start_capture_event = StartCapture(num_images=num_images, detector=camera) - readout_images_event = ReadoutData(num_images=num_images, detector=camera, + start_capture_event = StartCapture(num_blocks=num_images, detector=camera) + readout_images_event = ReadoutData(num_blocks=num_images, detector=camera, data_coordinates_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler) _ = executor.submit(start_capture_event) diff --git a/docs/usage/backends/ndtiff_and_ram_backend.rst b/docs/usage/backends/ndtiff_and_ram_backend.rst new file mode 100644 index 0000000..94a5d51 --- /dev/null +++ b/docs/usage/backends/ndtiff_and_ram_backend.rst @@ -0,0 +1,51 @@ +.. _NDTiff and RAM backend: + +################################################################## +NDTiff and RAM backend +################################################################## + +`NDTiff `_ is a high-performance indexed Tiff format used to save image data in Micro-Manager. NDRAM is a light-weight in-memory storage class. Both implementations provide the same API for in-memory (NDRAMStorage) or file-based (NDTiffStorage) storage. + +`NDTiff `_ is a high-performance, indexed Tiff format that can be written to local disk or network storage. It is one of the formats used in Micro-Manager. NDRAM is a simple in-memory storage. Both share the same API, enabling easy switching between file-based and in-memory storage. + + +To install this backend: + +.. code-block:: bash + + pip install exengine[ndstorage] + +No further setup is required. + + +Usage +`````` +.. code-block:: python + + from exengine.storage_backends.ndtiff_and_ndram import NDRAMStorage, NDTiffStorage + from exengine.kernel.data_coords import DataCoordinates + import numpy as np + + # Choose storage type + storage = NDRAMStorage() # In-memory + # OR + storage = NDTiffStorage(directory="/path/to/save") # File-based + + # Store data + coords = DataCoordinates(time=1, channel="DAPI", z=0) + data = np.array([[1, 2], [3, 4]], dtype=np.uint16) + metadata = {"exposure": 100} + storage.put(coords, data, metadata) + + # Retrieve data + retrieved_data = storage.get_data(coords) + retrieved_metadata = storage.get_metadata(coords) + + # Check existence + if coords in storage: + print("Data exists") + + # Finalize and close + storage.finish() + storage.close() + diff --git a/docs/usage/installation.rst b/docs/usage/installation.rst index 78263c8..b5b1aa9 100644 --- a/docs/usage/installation.rst +++ b/docs/usage/installation.rst @@ -5,61 +5,21 @@ Installation and Setup Basic Installation ------------------ -To install ExEngine with all available backends, use pip: +ExEngine was design to support multiple device and data storage backends, either independently or in combination. Each backend has some adapter code that lives within the Exengine package, and (optionally) other dependencies that need to be installed separately. -.. code-block:: bash - - pip install "exengine[all]" - -The ``[all]`` option installs ExEngine with all available device and storage backend dependencies. - -Alternatively, if you only want to install specific backends, you can use: +To install ExEngine with all available backend adapters, use pip: .. code-block:: bash - pip install "exengine[micromanager]" - -This will install ExEngine with only the Micro-Manager device backend. - - -Backend-Specific Setup ----------------------- -Some backends may require additional setup: - -Micro-Manager -^^^^^^^^^^^^^ -1. Install Micro-Manager: - - .. code-block:: python - - from mmpycorex import download_and_install_mm - download_and_install_mm() - -2. Configure your devices: - - After installation, you need to open Micro-Manager and create a configuration file for your devices. This process involves setting up and saving the hardware configuration for your specific microscope setup. - - For detailed instructions on creating a Micro-Manager configuration file, please refer to the `Micro-Manager Configuration Guide `_. - -3. Launch Micro-Manager pointing to the instance you installed and load the config file you made: - - .. code-block:: python - - from mmpycorex import create_core_instance - - create_core_instance(mm_app_path='/path/to/micro-manager', mm_config_path='name_of_config.cfg') - - For testing purposes, if you call ``create_core_instance()`` with no arguments it will default to the default installation path of ``download_and_install_mm()`` and the Micro-Manager demo configuration file. - + pip install "exengine[all]" -4. Verify the setup: +Note that even if you install all backends, you will still may need to install additional dependencies for some backends. - .. code-block:: python +Refer to :ref:`backends` for more information on additional setup needed for other Device and DataStorage backends. - from mmpycorex import Core +If you you only want to install specific backends, you can use: - core = Core() +.. code-block:: bash - print(core.get_loaded_devices()) + pip install "exengine[backend_name1, backend_name2]" - This should print a list of all devices loaded from your configuration file. diff --git a/src/exengine/examples/micromanager_example.py b/src/exengine/examples/micromanager_example.py index 62e71e1..1a65e96 100644 --- a/src/exengine/examples/micromanager_example.py +++ b/src/exengine/examples/micromanager_example.py @@ -23,8 +23,8 @@ num_images = 100 data_handler = DataHandler(storage=NDRAMStorage()) -start_capture_event = StartCapture(num_images=num_images, detector=camera) -readout_images_event = ReadoutData(number=num_images, detector=camera, +start_capture_event = StartCapture(num_blocks=num_images, detector=camera) +readout_images_event = ReadoutData(num_blocks=num_images, detector=camera, data_coordinates_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler) executor.submit(start_capture_event) diff --git a/src/exengine/integration_tests/test_imports.py b/src/exengine/integration_tests/test_imports.py index b606de4..87eb27c 100644 --- a/src/exengine/integration_tests/test_imports.py +++ b/src/exengine/integration_tests/test_imports.py @@ -33,5 +33,12 @@ def test_mm_imports(): try: from exengine.backends.micromanager import (MicroManagerDevice, MicroManagerCamera, MicroManagerSingleAxisStage, MicroManagerXYStage) + except ImportError as e: + pytest.fail(f"Import failed for MicroManagerDevice: {e}") + + +def test_ndstorage_imports(): + try: + from exengine.storage_backends.ndtiff_and_ndram import NDTiffStorage, NDRAMStorage except ImportError as e: pytest.fail(f"Import failed for MicroManagerDevice: {e}") \ No newline at end of file diff --git a/src/exengine/storage_backends/ndtiff_and_ndram/__init__.py b/src/exengine/storage_backends/ndtiff_and_ndram/__init__.py index e69de29..1b5b434 100644 --- a/src/exengine/storage_backends/ndtiff_and_ndram/__init__.py +++ b/src/exengine/storage_backends/ndtiff_and_ndram/__init__.py @@ -0,0 +1 @@ +from .NDTiffandRAM import NDRAMStorage, NDTiffStorage \ No newline at end of file From f60a885f0d5af2b761493e914de34b2cd950585f Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:04:26 -0400 Subject: [PATCH 6/6] fix mm tests --- src/exengine/backends/micromanager/test/test_mm_camera.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/exengine/backends/micromanager/test/test_mm_camera.py b/src/exengine/backends/micromanager/test/test_mm_camera.py index daa03d8..fd80ddc 100644 --- a/src/exengine/backends/micromanager/test/test_mm_camera.py +++ b/src/exengine/backends/micromanager/test/test_mm_camera.py @@ -26,8 +26,8 @@ def capture_images(num_images, executor, camera): storage = NDRAMStorage() data_handler = DataHandler(storage=storage) - start_capture_event = StartCapture(num_images=num_images, detector=camera) - readout_images_event = ReadoutData(number=num_images, detector=camera, + start_capture_event = StartCapture(num_blocks=num_images, detector=camera) + readout_images_event = ReadoutData(num_blocks=num_images, detector=camera, data_coordinates_iterator=[DataCoordinates(time=t) for t in range(num_images)], data_handler=data_handler)