From be50bc311347b57f4a1eefdc8d3e772b6e8ef3bd Mon Sep 17 00:00:00 2001 From: Bongjin Koo Date: Wed, 18 May 2022 14:19:21 -0700 Subject: [PATCH] Issue #86: Add jupyter to environment.yml. --- environment.yml | 1 + ioSPI/datasets.py | 9 ++-- notebooks/download_and_upload_with_osf.ipynb | 54 ++++++++++++++------ tests/test_datasets.py | 11 +--- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/environment.yml b/environment.yml index ebcd47c..a867d44 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,7 @@ dependencies: - mrcfile - numba - numpy + - jupyter - h5py>=2.10.0 - pandas - pillow>=8.2.0 diff --git a/ioSPI/datasets.py b/ioSPI/datasets.py index 8ea891b..31c3215 100644 --- a/ioSPI/datasets.py +++ b/ioSPI/datasets.py @@ -65,7 +65,7 @@ def __init__( def ls(self): """List all files in the project.""" print(f"Listing files from OSF project: {self.project_id}...") - return subprocess.run( + file_list = subprocess.run( self.osfclient_command + "ls", shell=True, text=True, @@ -73,6 +73,8 @@ def ls(self): stdout=subprocess.PIPE, ).stdout + return io.StringIO(file_list).readlines() + def download(self, remote_path: str = None, local_path: str = None): """Download a file from an OSF project and save it locally. @@ -130,14 +132,13 @@ def upload(self, local_path: str = None, remote_path: str = None): full_remote_path = self.storage + "/" + remote_path print(f"Uploading {local_path} to {full_remote_path}...") - f = subprocess.run( + subprocess.run( self.osfclient_command + f"upload {local_path} " f"{full_remote_path}", shell=True, text=True, check=True, stdout=subprocess.PIPE, - ).stdout - print(io.StringIO(f).readlines()) + ) print("Done!") def remove(self, remote_path: str = None): diff --git a/notebooks/download_and_upload_with_osf.ipynb b/notebooks/download_and_upload_with_osf.ipynb index ef51b77..5e2ddd4 100644 --- a/notebooks/download_and_upload_with_osf.ipynb +++ b/notebooks/download_and_upload_with_osf.ipynb @@ -51,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "e74dd634", "metadata": { "pycharm": { @@ -114,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "febfdedd", "metadata": { "pycharm": { @@ -155,7 +155,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "id": "f4f5b054", "metadata": { "pycharm": { @@ -167,9 +167,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "Listing files from OSF project: xbr2m...\n", - "osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n" + "Listing files from OSF project: xbr2m...\n" ] + }, + { + "data": { + "text/plain": "['osfstorage/test_upload-FcEBX.txt\\n',\n 'osfstorage/Vtest_uploadptest_uploadptest_uploadTtest_uploadj.txt\\n',\n 'osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n']" + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -196,7 +203,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "id": "6f5270f4", "metadata": { "pycharm": { @@ -216,7 +223,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.22k/4.22k [00:00<00:00, 18.0Mbytes/s]\n" + "100%|██████████| 4.22k/4.22k [00:00<00:00, 27.0Mbytes/s]\n" ] } ], @@ -265,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "id": "74c6125e", "metadata": { "pycharm": { @@ -283,7 +290,7 @@ } ], "source": [ - "osf_project.upload(\"4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\", \"new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\")" + "osf_project.upload(local_path=\"4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\", remote_path=\"new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\")" ] }, { @@ -300,16 +307,22 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Listing files from OSF project: xbr2m...\n", - "osfstorage/new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n", - "osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n" + "Listing files from OSF project: xbr2m...\n" ] + }, + { + "data": { + "text/plain": "['osfstorage/new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n',\n 'osfstorage/test_upload-FcEBX.txt\\n',\n 'osfstorage/Vtest_uploadptest_uploadptest_uploadTtest_uploadj.txt\\n',\n 'osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n']" + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -340,7 +353,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "outputs": [ { "name": "stdout", @@ -375,15 +388,22 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Listing files from OSF project: xbr2m...\n", - "osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n" + "Listing files from OSF project: xbr2m...\n" ] + }, + { + "data": { + "text/plain": "['osfstorage/test_upload-FcEBX.txt\\n',\n 'osfstorage/Vtest_uploadptest_uploadptest_uploadTtest_uploadj.txt\\n',\n 'osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n']" + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ diff --git a/tests/test_datasets.py b/tests/test_datasets.py index f69f7e9..69d66ea 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -4,7 +4,6 @@ when tested non-locally. """ -import io import os import random import string @@ -83,13 +82,10 @@ def test_upload_valid(setup, set_file_path): setup.upload(set_file_path[0] + set_file_path[1], set_file_path[1]) file_exists = False file_list = setup.ls() - file_list = io.StringIO(file_list) - line = file_list.readline() - while line: + for line in file_list: file_exists = set_file_path[1] == line.split("/")[1].strip() if file_exists: break - line = file_list.readline() assert file_exists subprocess.run(f"rm {set_file_path[0]}{set_file_path[1]}", shell=True, check=True) @@ -131,13 +127,10 @@ def test_remove_valid(setup, set_file_path): setup.remove(set_file_path[1]) file_exists = False file_list = setup.ls() - file_list = io.StringIO(file_list) - line = file_list.readline() - while line: + for line in file_list: file_exists = set_file_path[1] == line.split("/")[1].strip() if file_exists: break - line = file_list.readline() assert not file_exists