Issue #86: Add jupyter to environment.yml.

compSPI · May 18, 2022 · be50bc3 · be50bc3
1 parent aa08d56
commit be50bc3
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 30 deletions.
diff --git a/environment.yml b/environment.yml
@@ -9,6 +9,7 @@ dependencies:
   - mrcfile
   - numba
   - numpy
+  - jupyter
   - h5py>=2.10.0
   - pandas
   - pillow>=8.2.0

diff --git a/ioSPI/datasets.py b/ioSPI/datasets.py
@@ -65,14 +65,16 @@ def __init__(
     def ls(self):
         """List all files in the project."""
         print(f"Listing files from OSF project: {self.project_id}...")
-        return subprocess.run(
+        file_list = subprocess.run(
             self.osfclient_command + "ls",
             shell=True,
             text=True,
             check=True,
             stdout=subprocess.PIPE,
         ).stdout
 
+        return io.StringIO(file_list).readlines()
+
     def download(self, remote_path: str = None, local_path: str = None):
         """Download a file from an OSF project and save it locally.
 
@@ -130,14 +132,13 @@ def upload(self, local_path: str = None, remote_path: str = None):
 
         full_remote_path = self.storage + "/" + remote_path
         print(f"Uploading {local_path} to {full_remote_path}...")
-        f = subprocess.run(
+        subprocess.run(
             self.osfclient_command + f"upload {local_path} " f"{full_remote_path}",
             shell=True,
             text=True,
             check=True,
             stdout=subprocess.PIPE,
-        ).stdout
-        print(io.StringIO(f).readlines())
+        )
         print("Done!")
 
     def remove(self, remote_path: str = None):

diff --git a/notebooks/download_and_upload_with_osf.ipynb b/notebooks/download_and_upload_with_osf.ipynb
@@ -51,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "id": "e74dd634",
    "metadata": {
     "pycharm": {
@@ -114,7 +114,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "id": "febfdedd",
    "metadata": {
     "pycharm": {
@@ -155,7 +155,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 3,
    "id": "f4f5b054",
    "metadata": {
     "pycharm": {
@@ -167,9 +167,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Listing files from OSF project: xbr2m...\n",
-      "osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n"
+      "Listing files from OSF project: xbr2m...\n"
      ]
+    },
+    {
+     "data": {
+      "text/plain": "['osfstorage/test_upload-FcEBX.txt\\n',\n 'osfstorage/Vtest_uploadptest_uploadptest_uploadTtest_uploadj.txt\\n',\n 'osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n']"
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -196,7 +203,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 4,
    "id": "6f5270f4",
    "metadata": {
     "pycharm": {
@@ -216,7 +223,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 4.22k/4.22k [00:00<00:00, 18.0Mbytes/s]\n"
+      "100%|██████████| 4.22k/4.22k [00:00<00:00, 27.0Mbytes/s]\n"
      ]
     }
    ],
@@ -265,7 +272,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 5,
    "id": "74c6125e",
    "metadata": {
     "pycharm": {
@@ -283,7 +290,7 @@
     }
    ],
    "source": [
-    "osf_project.upload(\"4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\", \"new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\")"
+    "osf_project.upload(local_path=\"4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\", remote_path=\"new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\")"
    ]
   },
   {
@@ -300,16 +307,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 6,
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Listing files from OSF project: xbr2m...\n",
-      "osfstorage/new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n",
-      "osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n"
+      "Listing files from OSF project: xbr2m...\n"
      ]
+    },
+    {
+     "data": {
+      "text/plain": "['osfstorage/new_version_4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n',\n 'osfstorage/test_upload-FcEBX.txt\\n',\n 'osfstorage/Vtest_uploadptest_uploadptest_uploadTtest_uploadj.txt\\n',\n 'osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n']"
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -340,7 +353,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 7,
    "outputs": [
     {
      "name": "stdout",
@@ -375,15 +388,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 8,
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Listing files from OSF project: xbr2m...\n",
-      "osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\n"
+      "Listing files from OSF project: xbr2m...\n"
      ]
+    },
+    {
+     "data": {
+      "text/plain": "['osfstorage/test_upload-FcEBX.txt\\n',\n 'osfstorage/Vtest_uploadptest_uploadptest_uploadTtest_uploadj.txt\\n',\n 'osfstorage/4v6x_randomrot_copy0_defocus3.0_yes_noise.txt\\n']"
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [

diff --git a/tests/test_datasets.py b/tests/test_datasets.py
@@ -4,7 +4,6 @@
 when tested non-locally.
 """
 
-import io
 import os
 import random
 import string
@@ -83,13 +82,10 @@ def test_upload_valid(setup, set_file_path):
     setup.upload(set_file_path[0] + set_file_path[1], set_file_path[1])
     file_exists = False
     file_list = setup.ls()
-    file_list = io.StringIO(file_list)
-    line = file_list.readline()
-    while line:
+    for line in file_list:
         file_exists = set_file_path[1] == line.split("/")[1].strip()
         if file_exists:
             break
-        line = file_list.readline()
 
     assert file_exists
     subprocess.run(f"rm {set_file_path[0]}{set_file_path[1]}", shell=True, check=True)
@@ -131,13 +127,10 @@ def test_remove_valid(setup, set_file_path):
     setup.remove(set_file_path[1])
     file_exists = False
     file_list = setup.ls()
-    file_list = io.StringIO(file_list)
-    line = file_list.readline()
-    while line:
+    for line in file_list:
         file_exists = set_file_path[1] == line.split("/")[1].strip()
         if file_exists:
             break
-        line = file_list.readline()
 
     assert not file_exists