Merge pull request #120 from fedorov/make-dest-dir

ENH: simplify download and create destination directory if needed

idc_index/cli.py

@@ -113,7 +113,7 @@ def set_log_level(log_level):
     "--dir-template",
     type=str,
     default=IDCClient.DOWNLOAD_HIERARCHY_DEFAULT,
-    help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to None all files will be downloaded to the download directory with no subdirectories.",
+    help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to empty string (\"\") all files will be downloaded to the download directory with no subdirectories.",
 )
 def download_from_selection(
     download_dir,
@@ -233,7 +233,7 @@ def download_from_selection(
     "--dir-template",
     type=str,
     default=IDCClient.DOWNLOAD_HIERARCHY_DEFAULT,
-    help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to None all files will be downloaded to the download directory with no subdirectories.",
+    help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to empty string (\"\") all files will be downloaded to the download directory with no subdirectories.",
 )
 def download_from_manifest(
     manifest_file,

idc_index/index.py

@@ -207,6 +207,16 @@ def get_idc_version():
         idc_version = Version(idc_index_data.__version__).major
         return f"v{idc_version}"
 
+    @staticmethod
+    def _check_create_directory(download_dir):
+        """
+        Mimic behavior of s5cmd and create the download directory if it does not exist
+        """
+        download_dir = Path(download_dir)
+        download_dir.mkdir(parents=True, exist_ok=True)
+
+        return str(download_dir.resolve())
+
     def fetch_index(self, index) -> None:
         """
         Downloads requested index.
@@ -1333,9 +1343,7 @@ def download_from_manifest(
             ValueError: If the download directory does not exist.
         """
 
-        downloadDir = os.path.abspath(downloadDir).replace("\\", "/")
-        if not os.path.exists(downloadDir):
-            raise ValueError("Download directory does not exist.")
+        downloadDir = self._check_create_directory(downloadDir)
 
         # validate the manifest
         (
@@ -1512,9 +1520,7 @@ def download_from_selection(
 
         """
 
-        downloadDir = os.path.abspath(downloadDir).replace("\\", "/")
-        if not os.path.exists(downloadDir):
-            raise ValueError("Download directory does not exist.")
+        downloadDir = self._check_create_directory(downloadDir)
 
         result_df = self._safe_filter_by_selection(
             self.index,