From 6749faf179896fc5379b1dc8a5f2445c23f01c3f Mon Sep 17 00:00:00 2001 From: epnev Date: Thu, 21 Feb 2019 13:31:59 -0500 Subject: [PATCH 1/4] added info about threshold parameters when plotting contours --- caiman/source_extraction/cnmf/estimates.py | 35 +++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/caiman/source_extraction/cnmf/estimates.py b/caiman/source_extraction/cnmf/estimates.py index a784554a3..e6a1824f5 100644 --- a/caiman/source_extraction/cnmf/estimates.py +++ b/caiman/source_extraction/cnmf/estimates.py @@ -152,7 +152,7 @@ def __init__(self, A=None, b=None, C=None, f=None, R=None, dims=None): def plot_contours(self, img=None, idx=None, crd=None, thr_method='max', - thr='0.2', display_numbers=True): + thr='0.2', display_numbers=True, params=None): """view contours of all spatial footprints. Args: @@ -169,6 +169,8 @@ def plot_contours(self, img=None, idx=None, crd=None, thr_method='max', threshold value display_numbers : bool flag for displaying the id number of each contour + params : params object + set of dictionary containing the various parameters """ if 'csc_matrix' not in str(type(self.A)): self.A = scipy.sparse.csc_matrix(self.A) @@ -176,7 +178,12 @@ def plot_contours(self, img=None, idx=None, crd=None, thr_method='max', img = np.reshape(np.array(self.A.mean(1)), self.dims, order='F') if self.coordinates is None: # not hasattr(self, 'coordinates'): self.coordinates = caiman.utils.visualization.get_contours(self.A, self.dims, thr=thr, thr_method=thr_method) - plt.figure() + plt.figure() + if params is not None: + plt.suptitle('min_SNR=%1.2f, rval_thr=%1.2f, use_cnn=%i' + %(params.quality['SNR_lowest'], + params.quality['rval_thr'], + int(params.quality['use_cnn']))) if idx is None: caiman.utils.visualization.plot_contours(self.A, img, coordinates=self.coordinates, display_numbers=display_numbers) @@ -200,7 +207,7 @@ def plot_contours(self, img=None, idx=None, crd=None, thr_method='max', return self def plot_contours_nb(self, img=None, idx=None, crd=None, thr_method='max', - thr='0.2'): + thr='0.2', params=None): """view contours of all spatial footprints (notebook environment). Args: @@ -215,6 +222,8 @@ def plot_contours_nb(self, img=None, idx=None, crd=None, thr_method='max', thresholding method for computing contours ('max', 'nrg') thr : float threshold value + params : params object + set of dictionary containing the various parameters """ try: import bokeh @@ -230,6 +239,12 @@ def plot_contours_nb(self, img=None, idx=None, crd=None, thr_method='max', self.dims[1], coordinates=self.coordinates, thr_method=thr_method, thr=thr, show=False) p.title.text = 'Contour plots of found components' + if params is not None: + p.xaxis.axis_label = '''\ + min_SNR={min_SNR}, rval_thr={rval_thr}, use_cnn={use_cnn}\ + '''.format(min_SNR=params.quality['SNR_lowest'], + rval_thr=params.quality['rval_thr'], + use_cnn=params.quality['use_cnn']) bokeh.plotting.show(p) else: if not isinstance(idx, list): @@ -243,6 +258,12 @@ def plot_contours_nb(self, img=None, idx=None, crd=None, thr_method='max', p1.plot_width = 450 p1.plot_height = 450 * self.dims[0] // self.dims[1] p1.title.text = "Accepted Components" + if params is not None: + p1.xaxis.axis_label = '''\ + min_SNR={min_SNR}, rval_thr={rval_thr}, use_cnn={use_cnn}\ + '''.format(min_SNR=params.quality['SNR_lowest'], + rval_thr=params.quality['rval_thr'], + use_cnn=params.quality['use_cnn']) bad = list(set(range(self.A.shape[1])) - set(idx)) p2 = caiman.utils.visualization.nb_plot_contour(img, self.A[:, bad], self.dims[0], self.dims[1], coordinates=coor_b, @@ -250,12 +271,18 @@ def plot_contours_nb(self, img=None, idx=None, crd=None, thr_method='max', p2.plot_width = 450 p2.plot_height = 450 * self.dims[0] // self.dims[1] p2.title.text = 'Rejected Components' + if params is not None: + p2.xaxis.axis_label = '''\ + min_SNR={min_SNR}, rval_thr={rval_thr}, use_cnn={use_cnn}\ + '''.format(min_SNR=params.quality['SNR_lowest'], + rval_thr=params.quality['rval_thr'], + use_cnn=params.quality['use_cnn']) bokeh.plotting.show(bokeh.layouts.row(p1, p2)) except: print("Bokeh could not be loaded. Either it is not installed or you are not running within a notebook") print("Using non-interactive plot as fallback") self.plot_contours(img=img, idx=idx, crd=crd, thr_method=thr_method, - thr=thr) + thr=thr, params=params) return self def view_components(self, Yr=None, img=None, idx=None): From 86d8de2641ead2328452eaf6745f4d6fcb830005 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Thu, 21 Feb 2019 14:27:46 -0500 Subject: [PATCH 2/4] README: Better docs on deleting vs2015_compiler_vars.bat on Windows during install --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b7bb4d3d4..0158188ae 100755 --- a/README.md +++ b/README.md @@ -121,9 +121,12 @@ Use the following menu item to launch a anaconda-enabled command prompt: start>p ``` Then run ```caimanmanager``` as described above to make a data directory. +If you have trouble activating the caiman environment (with your command shell disappearing when you try): + * The default binding between Visual Studio and Conda can be broken by subtle version differences that mean the startup script for your environment may crash + * You can fix this by doing a file search under your Conda installation directory for vs2015_compiler_vars.bat and deleting it. Find the file with a windows file search (under the start menu) for that filename, and delete the one with conda\envs\caiman as part of the path + Things that can make your life easier on windows (but which take advanced Conda knowledge): * Finding the keras_activate.bat script for your environment and change the KERAS_BACKEND env var from theano to tensorflow - * Remove the vs2015_compiler_vars.bat from your environment setup Alternative environments: * [Using GPU](/README-GPU.md) From f3e6e0ee950c501ab69eb96765f44f06629f944d Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Thu, 21 Feb 2019 15:41:52 -0500 Subject: [PATCH 3/4] README/Install docs: Split off Windows instructions --- INSTALL-windows.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++ README.md | 34 ++++----------------------- 2 files changed, 63 insertions(+), 29 deletions(-) create mode 100755 INSTALL-windows.md diff --git a/INSTALL-windows.md b/INSTALL-windows.md new file mode 100755 index 000000000..7917e7bf1 --- /dev/null +++ b/INSTALL-windows.md @@ -0,0 +1,58 @@ +### Installation on Windows +The Windows installation process differs more widely from installation on Linux or MacOSX and has different issues you may run into. + +### Process + * Increase the maximum size of your pagefile to 64G or more (http://www.tomshardware.com/faq/id-2864547/manage-virtual-memory-pagefile-windows.html ) - The Windows memmap interface is sensitive to the maximum setting and leaving it at the default can cause errors when processing larger datasets + * Download and install Anaconda (Python 3.x, not 2.x) . Allow the installer to modify your PATH variable + * Use Conda to install git (With "conda install git") - use of another commandline git is acceptable, but may lead to issues depending on default settings + * Install Microsoft Build Tools for Visual Studio 2017 . Check the "Build Tools" box, and in the detailed view on the right check the "C/C++ CLI Tools" component too. The specifics of this occasionally change as Microsoft changes its products and website; you may need to go off-script. + +Use the following menu item to launch a anaconda-enabled command prompt: start>programs>anaconda3>anaconda prompt +From that prompt. issue the following commands (if you wish to use the dev branch, you may switch branches after the clone): + + ```bash + git clone https://github.com/flatironinstitute/CaImAn + cd CaImAn + conda env create -f environment.yml -n caiman + conda install -n caiman vs2017_win-64 + ``` + +At this point you will want to remove a startup script that visual studio made for your conda environment that can cause conda to crash while entering the caiman environment. Use the Windows find-file utility (under the Start Menu) to look for vs2015_compiler_vars.bat under your home directory. At least one copy should show up. Delete the version that has conda\envs\caiman as part of its location. You may also want to do a search for keras_activate.bat under your home directory, find the one in conda\envs\caiman, and edit it so KERAS_BACKEND is set to tensorflow rather than theano. You may then continue the installation. + + ```bash + activate caiman + pip install . (OR pip install -e . if you want to develop code) + copy caimanmanager.py .. + conda install numba + cd .. + ``` + +### Setting up a data directory with caimanmanager + +Now that you have stepped out of the caiman source directory, you are ready to make a data directory with code samples and datasets. You will not use the source tree directory any more. + + ``` + caimanmanager.py install + ``` + or + ``` + python caimanmanager.py install --inplace + ``` + if you used "pip install -e ." + +This will place that directory under your home directory in a directory called caiman_data. If you have, some of the demos or datafiles may have changed since your last install, to follow API changes. You can check to see if they have by doing `caimanmanager.py check`. If they have not, you may keep using them. If they have, we recommend moving your old caiman data directory out of the way (or just remove them if you have no precious data) and doing a new data install as per above. + +If you prefer to manage this information somewhere else, the `CAIMAN_DATA` environment variable can be set to customise it. The caimanmanager tool and other libraries will respect that. + +### Setting up environment variables + +To make the package work *efficiently* and eliminate "crosstalk" between different processes, run these commands before launching Python: + + ```bash + set MKL_NUM_THREADS=1 + set OPENBLAS_NUM_THREADS=1 + set KERAS_BACKEND=tensorflow + ``` + +The commands should be run every time you enter the caiman conda environment. We recommend you save these values inside your environment so you do not have to repeat this process every time. You can do this by following the instructions [here](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#saving-environment-variables). + diff --git a/README.md b/README.md index 0158188ae..3aafc3ac2 100755 --- a/README.md +++ b/README.md @@ -57,9 +57,12 @@ Also, if you want to install new packages into your conda environment for CaImAn ``` You will notice that any packages installed this way will mention, in their listing, that they're from conda-forge, with none of them having a blank origin. If you fail to do this, differences between how packages are built in conda-forge versus the default conda channels may mean that some packages (e.g. OpenCV) stop working despite showing as installed. +### Installation on Windows +On Windows, please follow the install instructions [here](/INSTALL-windows.md) . + ### Installation on Mac or Linux - * Download and install Anaconda or Miniconda (Python 3.6 version recommended) + * Download and install Anaconda or Miniconda (Python 3.x version) ```bash git clone https://github.com/flatironinstitute/CaImAn @@ -81,7 +84,7 @@ To make the package work *efficiently* and eliminate "crosstalk" between differe export MKL_NUM_THREADS=1 export OPENBLAS_NUM_THREADS=1 ``` -On Windows, replace the word "export" with the word "set" to achieve the same effect. The commands should be run every time before launching python. It is recommended that you save these values inside your environment so you don't have to repeat this process every time. You can do this by following the instructions [here](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#saving-environment-variables). +The commands should be run every time before launching python. It is recommended that you save these values inside your environment so you don't have to repeat this process every time. You can do this by following the instructions [here](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#saving-environment-variables). ### Setting up caimanmanager @@ -99,34 +102,7 @@ This will place that directory under your home directory in a directory called c If you prefer to manage this information somewhere else, the `CAIMAN_DATA` environment variable can be set to customise it. The caimanmanager tool and other libraries will respect that. -### Installation on Windows - * Increase the maximum size of your pagefile to 64G or more (http://www.tomshardware.com/faq/id-2864547/manage-virtual-memory-pagefile-windows.html ) - The Windows memmap interface is sensitive to the maximum setting and leaving it at the default can cause errors when processing larger datasets - * Download and install Anaconda (Python 3.6 recommended) . We recommend telling conda to modify your PATH variable (it is a checkbox during Anaconda install, off by default) - * Use Conda to install git (With "conda install git") - use of another commandline git is acceptable, but may lead to issues depending on default settings - * Microsoft Build Tools for Visual Studio 2017 . Check the "Build Tools" box, and in the detailed view on the right check the "C/C++ CLI Tools" component too. The specifics of this occasionally change as Microsoft changes its products and website; you may need to go off-script. - * As of this writing, there is an upstream versioning issue between scikit and numpy on Windows. If you get a runtime error that skimage/tifffile is compiled against API version 0xc but you have 0xb, you will need to manually upgrade numpy in your conda environment (which may force other version changes). - -Use the following menu item to launch a anaconda-enabled command prompt: start>programs>anaconda3>anaconda prompt - - ```bash - git clone https://github.com/flatironinstitute/CaImAn - cd CaImAn - conda env create -f environment.yml -n caiman - conda install -n caiman vs2017_win-64 - activate caiman - pip install . (OR pip install -e . if you want to develop code) - copy caimanmanager.py .. - conda install numba - cd .. - ``` -Then run ```caimanmanager``` as described above to make a data directory. - -If you have trouble activating the caiman environment (with your command shell disappearing when you try): - * The default binding between Visual Studio and Conda can be broken by subtle version differences that mean the startup script for your environment may crash - * You can fix this by doing a file search under your Conda installation directory for vs2015_compiler_vars.bat and deleting it. Find the file with a windows file search (under the start menu) for that filename, and delete the one with conda\envs\caiman as part of the path -Things that can make your life easier on windows (but which take advanced Conda knowledge): - * Finding the keras_activate.bat script for your environment and change the KERAS_BACKEND env var from theano to tensorflow Alternative environments: * [Using GPU](/README-GPU.md) From f9fa2e2bf632ec38d4a7e456abf0f6da0b142f1f Mon Sep 17 00:00:00 2001 From: epnev Date: Thu, 21 Feb 2019 17:07:06 -0500 Subject: [PATCH 4/4] fixing issue #470 --- demos/general/demo_pipeline.py | 4 ++-- demos/notebooks/demo_pipeline.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/demos/general/demo_pipeline.py b/demos/general/demo_pipeline.py index 2051a2e7c..704677285 100755 --- a/demos/general/demo_pipeline.py +++ b/demos/general/demo_pipeline.py @@ -190,7 +190,7 @@ def main(): # First extract spatial and temporal components on patches and combine them # for this step deconvolution is turned off (p=0) - opts.set('temporal', {'p': 0}) + opts.change_params({'p': 0}) cnm = cnmf.CNMF(n_processes, params=opts, dview=dview) cnm = cnm.fit(images) @@ -207,7 +207,7 @@ def main(): plt.title('Contour plots of found components') # %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution - cnm.params.set('temporal', {'p': p}) + cnm.params.change_params({'p': p}) cnm2 = cnm.refit(images, dview=dview) # %% COMPONENT EVALUATION # the components are evaluated in three ways: diff --git a/demos/notebooks/demo_pipeline.ipynb b/demos/notebooks/demo_pipeline.ipynb index c9320c8d0..3c0188a51 100644 --- a/demos/notebooks/demo_pipeline.ipynb +++ b/demos/notebooks/demo_pipeline.ipynb @@ -357,7 +357,7 @@ "\n", "# First extract spatial and temporal components on patches and combine them\n", "# for this step deconvolution is turned off (p=0)\n", - "opts.set('temporal', {'p': 0})\n", + "opts.change_params({'p': 0})\n", "cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)\n", "cnm = cnm.fit(images)" ] @@ -419,7 +419,7 @@ "source": [ "%%capture\n", "#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution \n", - "cnm.params.set('temporal', {'p': p})\n", + "cnm.params.change_params({'p': p})\n", "cnm2 = cnm.refit(images, dview=dview)" ] },