From f070226ddc3d2c868f2c44a8e8c9820dcb20b611 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 3 May 2024 12:01:16 -0400 Subject: [PATCH 01/11] VERSION to 1.11.1 after release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1cac385c6..720c7384c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.11.1 From 084e44a058c28c853142be8d531227dc8bc64ae9 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 3 May 2024 12:27:25 -0400 Subject: [PATCH 02/11] See if updating to latest github actions for CI is easy (maybe will fix environment file selection?) --- .github/workflows/run_tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 2904dacc1..158d251ad 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -19,15 +19,16 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} environment-file: environment-minimal.yml activate-environment: caiman + conda-solver: libmamba - name: Install OS Dependencies shell: bash -l {0} From dea8431846a05f2633f5ed99e37011160cd8e439 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 7 May 2024 16:37:26 -0400 Subject: [PATCH 03/11] Fix code goof that a random bot found #1345 --- caiman/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/paths.py b/caiman/paths.py index 05abfb336..91dc1d0bf 100644 --- a/caiman/paths.py +++ b/caiman/paths.py @@ -38,7 +38,7 @@ def get_tempdir() -> str: if os.path.isdir(os.environ['CAIMAN_TEMP']): return os.environ['CAIMAN_TEMP'] else: - logging.warning(f"CAIMAN_TEMP is set to nonexistent directory {os.environment['CAIMAN_TEMP']}. Ignoring") + logging.warning(f"CAIMAN_TEMP is set to nonexistent directory {os.environ['CAIMAN_TEMP']}. Ignoring") temp_under_data = os.path.join(caiman_datadir(), "temp") if not os.path.isdir(temp_under_data): logging.warning(f"Default temporary dir {temp_under_data} does not exist, creating") From aa65ab3da1fc1f857b09bbb613df3a10e4e9e344 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Sun, 12 May 2024 16:01:38 -0400 Subject: [PATCH 04/11] Fix map_async call on DirectView causing error --- caiman/source_extraction/cnmf/spatial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/source_extraction/cnmf/spatial.py b/caiman/source_extraction/cnmf/spatial.py index f8b1a833d..045027c8a 100644 --- a/caiman/source_extraction/cnmf/spatial.py +++ b/caiman/source_extraction/cnmf/spatial.py @@ -504,7 +504,7 @@ def threshold_components(A, dims, medw=None, thr_method='max', maxthr=0.1, nrgth res = dview.map_async( threshold_components_parallel, pars).get(4294967) else: - res = dview.map_async(threshold_components_parallel, pars) + res = dview.map_sync(threshold_components_parallel, pars) else: res = list(map(threshold_components_parallel, pars)) From dbd27dbc136f06a22c3b8628707068958313c36d Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Wed, 15 May 2024 22:44:14 -0400 Subject: [PATCH 05/11] Clean up special cases for loading from HDF5 and fix None handling --- caiman/utils/utils.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index caf82d906..3fe65ea44 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -561,25 +561,16 @@ def recursively_load_dict_contents_from_group(h5file:h5py.File, path:str) -> dic for key, item in h5file[path].items(): if isinstance(item, h5py._hl.dataset.Dataset): - val_set = np.nan - if isinstance(item[()], str): - if item[()] == 'NoneType': - ans[key] = None - else: - ans[key] = item[()] - - elif key in ['dims', 'medw', 'sigma_smooth_snmf', 'dxy', 'max_shifts', 'strides', 'overlaps']: - if isinstance(item[()], np.ndarray): - ans[key] = tuple(item[()]) - else: - ans[key] = item[()] + val = item[()] + if val == 'NoneType' or val == b'NoneType': + ans[key] = None + elif key in ['dims', 'medw', 'sigma_smooth_snmf', + 'dxy', 'max_shifts', 'strides', 'overlaps'] and isinstance(val, np.ndarray): + ans[key] = tuple(val) + elif isinstance(val, np.bool_): # sigh + ans[key] = bool(val) else: - if isinstance(item[()], np.bool_): # sigh - ans[key] = bool(item[()]) - else: - ans[key] = item[()] - if isinstance(ans[key], bytes) and ans[key] == b'NoneType': - ans[key] = None + ans[key] = item[()] elif isinstance(item, h5py._hl.group.Group): if key in ('A', 'W', 'Ab', 'downscale_matrix', 'upscale_matrix'): From 09a8a0273a04d5d23516113b3f0fbf4618c2b9fe Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Wed, 15 May 2024 22:51:49 -0400 Subject: [PATCH 06/11] Fix invalid comparison for non-scalars --- caiman/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index 3fe65ea44..e0dd46f15 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -562,7 +562,7 @@ def recursively_load_dict_contents_from_group(h5file:h5py.File, path:str) -> dic for key, item in h5file[path].items(): if isinstance(item, h5py._hl.dataset.Dataset): val = item[()] - if val == 'NoneType' or val == b'NoneType': + if isinstance(val, str) and val == 'NoneType' or isinstance(val, bytes) and val == b'NoneType': ans[key] = None elif key in ['dims', 'medw', 'sigma_smooth_snmf', 'dxy', 'max_shifts', 'strides', 'overlaps'] and isinstance(val, np.ndarray): From ae1fe4e4a47b96ad254330d96221eb1135d246fc Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Thu, 30 May 2024 22:49:28 -0400 Subject: [PATCH 07/11] Pin tensorflow below 2.16, address #1355 --- environment-minimal.yml | 2 +- environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment-minimal.yml b/environment-minimal.yml index 6ff26de47..a7d097ed0 100644 --- a/environment-minimal.yml +++ b/environment-minimal.yml @@ -23,7 +23,7 @@ dependencies: - scikit-image >=0.19.0 - scikit-learn >=1.2 - scipy >= 1.10.1 -- tensorflow >=2.4.0 +- tensorflow >=2.4.0,<2.16 - tifffile - tqdm - zarr diff --git a/environment.yml b/environment.yml index 6ae3e04e5..6534f39a5 100644 --- a/environment.yml +++ b/environment.yml @@ -30,7 +30,7 @@ dependencies: - scikit-image >=0.19.0 - scikit-learn >=1.2 - scipy >= 1.10.1 -- tensorflow >=2.4.0 +- tensorflow >=2.4.0,<2.16 - tifffile - tk - tqdm From b506e0222b83fadc8708fbb8adc47e96cb1ae2fc Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 31 May 2024 12:29:46 -0400 Subject: [PATCH 08/11] caimanmanager: Detect and adjust behaviour for editable installs --- caiman/caimanmanager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/caiman/caimanmanager.py b/caiman/caimanmanager.py index c2d57b8c0..912ad9db0 100755 --- a/caiman/caimanmanager.py +++ b/caiman/caimanmanager.py @@ -3,6 +3,7 @@ import argparse import filecmp import glob +import json import os import platform import psutil @@ -52,6 +53,15 @@ def do_install_to(targdir: str, inplace: bool = False, force: bool = False) -> None: global sourcedir_base + + try: + import importlib_metadata + # A lot can change upstream with this code; I hope the APIs are stable, but just in case, make this best-effort + if json.loads(importlib_metadata.Distribution.from_name('caiman').read_text('direct_url.json'))['dir_info']['editable']: + inplace = True + except: + pass + ignore_pycache=shutil.ignore_patterns('__pycache__') if os.path.isdir(targdir) and not force: raise Exception(targdir + " already exists. You may move it out of the way, remove it, or use --force") From be70e72c7cb6b5bccd9af7c4605b4ed17e27c25c Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 31 May 2024 12:31:40 -0400 Subject: [PATCH 09/11] Improve messaging around editable installs --- caiman/caimanmanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/caiman/caimanmanager.py b/caiman/caimanmanager.py index 912ad9db0..067398ce7 100755 --- a/caiman/caimanmanager.py +++ b/caiman/caimanmanager.py @@ -55,6 +55,7 @@ def do_install_to(targdir: str, inplace: bool = False, force: bool = False) -> N global sourcedir_base try: + print("If you did an editable install (with -e), the install must happen within the source tree; otherwise, it must not") import importlib_metadata # A lot can change upstream with this code; I hope the APIs are stable, but just in case, make this best-effort if json.loads(importlib_metadata.Distribution.from_name('caiman').read_text('direct_url.json'))['dir_info']['editable']: From d0a5eb3a37094d1ab805247480220984925cf47f Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 31 May 2024 14:59:29 -0400 Subject: [PATCH 10/11] chdir for the user if needed --- caiman/caimanmanager.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/caiman/caimanmanager.py b/caiman/caimanmanager.py index 067398ce7..67e7af242 100755 --- a/caiman/caimanmanager.py +++ b/caiman/caimanmanager.py @@ -55,13 +55,16 @@ def do_install_to(targdir: str, inplace: bool = False, force: bool = False) -> N global sourcedir_base try: - print("If you did an editable install (with -e), the install must happen within the source tree; otherwise, it must not") + import importlib import importlib_metadata # A lot can change upstream with this code; I hope the APIs are stable, but just in case, make this best-effort if json.loads(importlib_metadata.Distribution.from_name('caiman').read_text('direct_url.json'))['dir_info']['editable']: inplace = True + cwd = os.getcwd() + os.chdir(str(importlib.resources.files('caiman').joinpath('..'))) + print(f"Used editable fallback, entered {os.getcwd()} directory") except: - pass + print("Did not use editable fallback") ignore_pycache=shutil.ignore_patterns('__pycache__') if os.path.isdir(targdir) and not force: @@ -88,6 +91,8 @@ def do_install_to(targdir: str, inplace: bool = False, force: bool = False) -> N with open(os.path.join(targdir, 'RELEASE'), 'w') as verfile_fh: print(f"Version:{caiman.__version__}", file=verfile_fh) print("Installed " + targdir) + if cwd is not None: + os.chdir(cwd) def do_check_install(targdir: str, inplace: bool = False) -> None: From ddca9828f3c9f52ee3911a3afb70bf757c3ad684 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 31 May 2024 15:05:59 -0400 Subject: [PATCH 11/11] caimanmanager: "is None" still can't be used on a variable if it was created in a branch, so assign =None --- caiman/caimanmanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/caiman/caimanmanager.py b/caiman/caimanmanager.py index 67e7af242..62580930c 100755 --- a/caiman/caimanmanager.py +++ b/caiman/caimanmanager.py @@ -53,6 +53,7 @@ def do_install_to(targdir: str, inplace: bool = False, force: bool = False) -> None: global sourcedir_base + cwd = None # Assigning so it exists to avoid UnboundLocalError try: import importlib