diff --git a/idc_index/cli.py b/idc_index/cli.py index 09fed46..3171276 100644 --- a/idc_index/cli.py +++ b/idc_index/cli.py @@ -113,7 +113,7 @@ def set_log_level(log_level): "--dir-template", type=str, default=IDCClient.DOWNLOAD_HIERARCHY_DEFAULT, - help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to None all files will be downloaded to the download directory with no subdirectories.", + help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to empty string (\"\") all files will be downloaded to the download directory with no subdirectories.", ) def download_from_selection( download_dir, @@ -233,7 +233,7 @@ def download_from_selection( "--dir-template", type=str, default=IDCClient.DOWNLOAD_HIERARCHY_DEFAULT, - help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to None all files will be downloaded to the download directory with no subdirectories.", + help="Download directory hierarchy template. This variable defines the folder hierarchy for the organizing the downloaded files in downloadDirectory. Defaults to index.DOWNLOAD_HIERARCHY_DEFAULT set to %collection_id/%PatientID/%StudyInstanceUID/%Modality_%SeriesInstanceUID. The template string can be built using a combination of selected metadata attributes (PatientID, collection_id, Modality, StudyInstanceUID, SeriesInstanceUID) that must be prefixed by '%'. The following special characters can be used as separators: '-' (hyphen), '/' (slash for subdirectories), '_' (underscore). When set to empty string (\"\") all files will be downloaded to the download directory with no subdirectories.", ) def download_from_manifest( manifest_file, diff --git a/idc_index/index.py b/idc_index/index.py index 2d1c59f..0524584 100644 --- a/idc_index/index.py +++ b/idc_index/index.py @@ -207,6 +207,16 @@ def get_idc_version(): idc_version = Version(idc_index_data.__version__).major return f"v{idc_version}" + @staticmethod + def _check_create_directory(download_dir): + """ + Mimic behavior of s5cmd and create the download directory if it does not exist + """ + download_dir = Path(download_dir) + download_dir.mkdir(parents=True, exist_ok=True) + + return str(download_dir.resolve()) + def fetch_index(self, index) -> None: """ Downloads requested index. @@ -1333,9 +1343,7 @@ def download_from_manifest( ValueError: If the download directory does not exist. """ - downloadDir = os.path.abspath(downloadDir).replace("\\", "/") - if not os.path.exists(downloadDir): - raise ValueError("Download directory does not exist.") + downloadDir = self._check_create_directory(downloadDir) # validate the manifest ( @@ -1512,9 +1520,7 @@ def download_from_selection( """ - downloadDir = os.path.abspath(downloadDir).replace("\\", "/") - if not os.path.exists(downloadDir): - raise ValueError("Download directory does not exist.") + downloadDir = self._check_create_directory(downloadDir) result_df = self._safe_filter_by_selection( self.index,