From 98281ee481da48dcdb39d6f44d76a87c30ebdc99 Mon Sep 17 00:00:00 2001 From: RyanSpies-NOAA Date: Mon, 30 Oct 2023 17:35:34 +0000 Subject: [PATCH 01/19] Fixes for inundate_nation to create boolean rasters --- config/params_template.env | 2 +- tools/inundate_nation.py | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/config/params_template.env b/config/params_template.env index 6c2cb1687..dd0ab4499 100644 --- a/config/params_template.env +++ b/config/params_template.env @@ -47,7 +47,7 @@ export src_subdiv_toggle="True" # text to append to output log and hydrotable file names (use for testing/debugging) export vrough_suffix="" # input file location with nwm feature_id and channel roughness and overbank roughness attributes -export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv" +export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_025_05.csv" #### SRC calibration variables # input file location with nwm feature_id and recurrence flow values diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index fe8680418..e915f602d 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -93,19 +93,13 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print("Performing bool mosaic process...") logging.info("Performing bool mosaic process...") - output_bool_dir = os.path.join(output_dir, "bool_temp") - if not os.path.exists(output_bool_dir): - os.mkdir(output_bool_dir) - else: - # we need to empty it. we will kill it and remake it (using rmtree to force it) - shutil.rmtree(output_bool_dir, ignore_errors=True) - os.mkdir(output_bool_dir) + output_bool_dir = output_dir procs_list = [] for rasfile in os.listdir(magnitude_output_dir): if rasfile.endswith(".tif") and "extent" in rasfile: # p = magnitude_output_dir + rasfile - procs_list.append([magnitude_output_dir, rasfile, output_bool_dir]) + procs_list.append([magnitude_output_dir, rasfile, output_bool_dir, fim_version]) # Multiprocess --> create boolean inundation rasters for all hucs if len(procs_list) > 0: @@ -117,7 +111,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(msg) # now cleanup the raw mosiac directories - shutil.rmtree(output_bool_dir, ignore_errors=True) + #shutil.rmtree(output_bool_dir, ignore_errors=True) # now cleanup the raw mosiac directories shutil.rmtree(magnitude_output_dir, ignore_errors=True) @@ -167,6 +161,7 @@ def create_bool_rasters(args): in_raster_dir = args[0] rasfile = args[1] output_bool_dir = args[2] + fim_version = args[3] print("Calculating boolean inundate raster: " + rasfile) p = in_raster_dir + os.sep + rasfile @@ -189,7 +184,7 @@ def create_bool_rasters(args): dtype="int8", compress="lzw", ) - with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, "w", **profile) as dst: + with rasterio.open(output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile) as dst: dst.write(array.astype(rasterio.int8)) From 74f8e23d877692a9d4ce55d86ecd86567b9992cd Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Fri, 3 Nov 2023 22:56:56 +0000 Subject: [PATCH 02/19] fixed bug with default huc lists --- tools/inundate_nation.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index e915f602d..b793e34ee 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -37,10 +37,12 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, + " max jobs will be used instead." ) + print("Script starting...") + print(f"Input FIM Directory is {fim_run_dir}") + fim_version = os.path.basename(os.path.normpath(fim_run_dir)) logging.info(f"Using fim version: {fim_version}") output_base_file_name = magnitude_key + "_" + fim_version - # print(output_base_file_name) __setup_logger(output_dir, output_base_file_name) @@ -65,7 +67,9 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, shutil.rmtree(magnitude_output_dir, ignore_errors=True) os.mkdir(magnitude_output_dir) - if huc_list is None: + print(f"type of huc_list is {type(huc_list)}") + + if huc_list == 'all' or len(huc_list) == 0: huc_list = [] for huc in os.listdir(fim_run_dir): # if ( @@ -78,11 +82,11 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, huc_list.append(huc) else: for huc in huc_list: - assert os.path.isdir( - fim_run_dir + os.sep + huc - ), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}' + huc_path = os.path.join(fim_run_dir, huc) + assert os.path.isdir(huc_path), f'ERROR: could not find the input fim_dir location: {huc_path}' + - print("Inundation raw mosaic outputs here: " + magnitude_output_dir) + print("Inundation raw mosaic outputs will saved here: " + magnitude_output_dir) run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number]) From 1dac8b3533fc6923d3e1ca85f36435d638cbcf5d Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Fri, 3 Nov 2023 23:08:05 +0000 Subject: [PATCH 03/19] took out debug line, added more prints --- tools/inundate_nation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index b793e34ee..e71ab408d 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -41,11 +41,12 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print(f"Input FIM Directory is {fim_run_dir}") fim_version = os.path.basename(os.path.normpath(fim_run_dir)) - logging.info(f"Using fim version: {fim_version}") - output_base_file_name = magnitude_key + "_" + fim_version + output_base_file_name = magnitude_key + "_" + fim_version __setup_logger(output_dir, output_base_file_name) + logging.info(f"Using fim version: {fim_version}") + start_dt = datetime.now() logging.info(f"Input FIM Directory: {fim_run_dir}") @@ -67,8 +68,6 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, shutil.rmtree(magnitude_output_dir, ignore_errors=True) os.mkdir(magnitude_output_dir) - print(f"type of huc_list is {type(huc_list)}") - if huc_list == 'all' or len(huc_list) == 0: huc_list = [] for huc in os.listdir(fim_run_dir): @@ -124,6 +123,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))) end_time = datetime.now() logging.info(fh.print_date_time_duration(start_dt, end_time)) + print(fh.print_date_time_duration(start_dt, end_time)) def run_inundation(args): @@ -198,6 +198,7 @@ def __setup_logger(output_folder_path, log_file_name_key): log_file_name = f"{log_file_name_key}-{file_dt_string}.log" log_file_path = os.path.join(output_folder_path, log_file_name) + print(f"log file will be at {log_file_path}") logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s") From b285864a63d8f2f6c983cc62ca39418165c28e1e Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Sat, 4 Nov 2023 00:01:42 +0000 Subject: [PATCH 04/19] fix some output print errors --- tools/inundate_mosaic_wrapper.py | 3 ++- tools/inundate_nation.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py index 1620a7820..0ceeb6fce 100644 --- a/tools/inundate_mosaic_wrapper.py +++ b/tools/inundate_mosaic_wrapper.py @@ -92,7 +92,8 @@ def produce_mosaicked_inundation( "Please lower the num_workers.".format(num_workers, total_cpus_available) ) - fh.vprint("Running inundate for " + huc + "...", verbose) + if len(hucs) == 1: + fh.vprint("Running inundate for " + huc + "...", verbose) # Call Inundate_gms map_file = Inundate_gms( diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index e71ab408d..403a6eac3 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -12,8 +12,8 @@ import rasterio from inundate_mosaic_wrapper import produce_mosaicked_inundation -from osgeo import gdal, ogr -from rasterio.merge import merge +#from osgeo import gdal, ogr +#from rasterio.merge import merge from utils.shared_functions import FIM_Helpers as fh from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv @@ -84,6 +84,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, huc_path = os.path.join(fim_run_dir, huc) assert os.path.isdir(huc_path), f'ERROR: could not find the input fim_dir location: {huc_path}' + huc_list.sort() print("Inundation raw mosaic outputs will saved here: " + magnitude_output_dir) From 930c1241a0f7a69ba0147bed760ae50d1cf09da3 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Sat, 4 Nov 2023 20:41:29 +0000 Subject: [PATCH 05/19] debugging --- tools/inundate_mosaic_wrapper.py | 3 --- tools/inundate_nation.py | 28 ++++++++++++++++------------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py index 0ceeb6fce..4271dbee1 100644 --- a/tools/inundate_mosaic_wrapper.py +++ b/tools/inundate_mosaic_wrapper.py @@ -92,9 +92,6 @@ def produce_mosaicked_inundation( "Please lower the num_workers.".format(num_workers, total_cpus_available) ) - if len(hucs) == 1: - fh.vprint("Running inundate for " + huc + "...", verbose) - # Call Inundate_gms map_file = Inundate_gms( hydrofabric_dir=hydrofabric_dir, diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index 403a6eac3..27b87ec83 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -12,8 +12,7 @@ import rasterio from inundate_mosaic_wrapper import produce_mosaicked_inundation -#from osgeo import gdal, ogr -#from rasterio.merge import merge +from rasterio.merge import merge from utils.shared_functions import FIM_Helpers as fh from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv @@ -37,15 +36,15 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, + " max jobs will be used instead." ) - print("Script starting...") + print() + print("Inundation Nation script starting...") print(f"Input FIM Directory is {fim_run_dir}") fim_version = os.path.basename(os.path.normpath(fim_run_dir)) - + logging.info(f"Using fim version: {fim_version}") output_base_file_name = magnitude_key + "_" + fim_version - __setup_logger(output_dir, output_base_file_name) - logging.info(f"Using fim version: {fim_version}") + __setup_logger(output_dir, output_base_file_name) start_dt = datetime.now() @@ -98,6 +97,13 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info("Performing bool mosaic process...") output_bool_dir = output_dir + + if not os.path.exists(output_bool_dir): + os.mkdir(output_bool_dir) + else: + # we need to empty it. we will kill it and remake it (using rmtree to force it) + shutil.rmtree(output_bool_dir, ignore_errors=True) + os.mkdir(output_bool_dir) procs_list = [] for rasfile in os.listdir(magnitude_output_dir): @@ -118,13 +124,12 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, #shutil.rmtree(output_bool_dir, ignore_errors=True) # now cleanup the raw mosiac directories - shutil.rmtree(magnitude_output_dir, ignore_errors=True) + # shutil.rmtree(magnitude_output_dir, ignore_errors=True) fh.print_current_date_time() logging.info(logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))) end_time = datetime.now() logging.info(fh.print_date_time_duration(start_dt, end_time)) - print(fh.print_date_time_duration(start_dt, end_time)) def run_inundation(args): @@ -156,8 +161,8 @@ def run_inundation(args): forecast, inundation_raster=inundation_raster, num_workers=job_number, - remove_intermediate=True, - verbose=True, + remove_intermediate=False, + verbose=False, is_mosaic_for_branches=True, ) @@ -199,7 +204,6 @@ def __setup_logger(output_folder_path, log_file_name_key): log_file_name = f"{log_file_name_key}-{file_dt_string}.log" log_file_path = os.path.join(output_folder_path, log_file_name) - print(f"log file will be at {log_file_path}") logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s") @@ -296,4 +300,4 @@ def __setup_logger(output_folder_path, log_file_name_key): args = vars(parser.parse_args()) - inundate_nation(**args) + inundate_nation(**args) \ No newline at end of file From 54de8ca81b117922cec0a5b1770180a6c856aaf3 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Sat, 4 Nov 2023 20:54:59 +0000 Subject: [PATCH 06/19] rolled most changes back to Ryan's orig fix --- tools/inundate_nation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index 27b87ec83..bc133da41 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -96,7 +96,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print("Performing bool mosaic process...") logging.info("Performing bool mosaic process...") - output_bool_dir = output_dir + output_bool_dir = os.path.join(output_dir, "bool_temp") if not os.path.exists(output_bool_dir): os.mkdir(output_bool_dir) From 03e4772bda7b39b5d3f9499d6625ef7604113156 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Sat, 4 Nov 2023 21:03:10 +0000 Subject: [PATCH 07/19] minor adj to temp dir --- tools/inundate_nation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index bc133da41..508b692cd 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -161,7 +161,7 @@ def run_inundation(args): forecast, inundation_raster=inundation_raster, num_workers=job_number, - remove_intermediate=False, + remove_intermediate=True, verbose=False, is_mosaic_for_branches=True, ) From bb18bf3ebb7268a13d560aaf6f308c962635d46c Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Tue, 14 Nov 2023 21:27:22 +0000 Subject: [PATCH 08/19] update notes in_nation --- tools/inundate_nation.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index 508b692cd..189ed960d 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -25,6 +25,9 @@ # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/' +# TODO: Nov 2023, Logging system appears to be not workign correctly. + + def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number): assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}" @@ -38,8 +41,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print() print("Inundation Nation script starting...") - print(f"Input FIM Directory is {fim_run_dir}") - + fim_version = os.path.basename(os.path.normpath(fim_run_dir)) logging.info(f"Using fim version: {fim_version}") output_base_file_name = magnitude_key + "_" + fim_version @@ -86,18 +88,21 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, huc_list.sort() print("Inundation raw mosaic outputs will saved here: " + magnitude_output_dir) + print() run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number]) + print() + # Perform mosaic operation if inc_mosaic: fh.print_current_date_time() logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S")) print("Performing bool mosaic process...") logging.info("Performing bool mosaic process...") - output_bool_dir = os.path.join(output_dir, "bool_temp") - + print(f"output_bool_dir is {output_bool_dir}") + if not os.path.exists(output_bool_dir): os.mkdir(output_bool_dir) else: @@ -120,8 +125,11 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print(msg) logging.info(msg) - # now cleanup the raw mosiac directories - #shutil.rmtree(output_bool_dir, ignore_errors=True) + # now cleanup the temp bool directory + shutil.rmtree(output_bool_dir, ignore_errors=True) + + else: + print("Skipping mosiaking") # now cleanup the raw mosiac directories # shutil.rmtree(magnitude_output_dir, ignore_errors=True) @@ -154,6 +162,12 @@ def run_inundation(args): inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif") print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude)) + print( + "This will take a long time depending on the number of HUCs. No progress bar will be shown." + " Once is gets to mosiacing (if applicable), screen output will exist. To see if the script has frozen," + " you should be able to watch the file system for some changes." + ) + print() produce_mosaicked_inundation( fim_run_dir, @@ -194,7 +208,9 @@ def create_bool_rasters(args): dtype="int8", compress="lzw", ) - with rasterio.open(output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile) as dst: + with rasterio.open( + output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile + ) as dst: dst.write(array.astype(rasterio.int8)) @@ -300,4 +316,4 @@ def __setup_logger(output_folder_path, log_file_name_key): args = vars(parser.parse_args()) - inundate_nation(**args) \ No newline at end of file + inundate_nation(**args) From dc0cf9eda7ef6c8b9a492a1d81e3416773916994 Mon Sep 17 00:00:00 2001 From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:51:05 -0700 Subject: [PATCH 09/19] Update inundate_nation.py --- tools/inundate_nation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index 189ed960d..a25b45feb 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -25,7 +25,7 @@ # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/' -# TODO: Nov 2023, Logging system appears to be not workign correctly. +# TODO: Nov 2023, Logging system appears to be not working correctly. def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number): From dc5a5c202a21f3d766d4b13204c495344d3ec1da Mon Sep 17 00:00:00 2001 From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:11:00 -0700 Subject: [PATCH 10/19] Update CHANGELOG.md --- docs/CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a6b6c8352..9d50db94d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,21 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.4.x.x - 2023-11-14 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026) + +A couple of directly related issues were fixed in this PR. +The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing. + +### Changes + +- `config`/`params_template.env`: Updated for a newer mannings global file. Changed and tested by Ryan Spies. +- `tools` + - `inundate_mosiac_wrapper.py`: Took out a misleading and non-required print statement. + - `inundate_nation.py`: As mentioned above. + +

+ + ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018) During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times. From d0a565d852cead46d4c522a4a3c098a0c8303c83 Mon Sep 17 00:00:00 2001 From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com> Date: Mon, 27 Nov 2023 08:50:39 -0700 Subject: [PATCH 11/19] Update params_template.env --- config/params_template.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/params_template.env b/config/params_template.env index dd0ab4499..6c2cb1687 100644 --- a/config/params_template.env +++ b/config/params_template.env @@ -47,7 +47,7 @@ export src_subdiv_toggle="True" # text to append to output log and hydrotable file names (use for testing/debugging) export vrough_suffix="" # input file location with nwm feature_id and channel roughness and overbank roughness attributes -export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_025_05.csv" +export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv" #### SRC calibration variables # input file location with nwm feature_id and recurrence flow values From 2569ca59ef8ee5781523d43a22c54571c7f617f9 Mon Sep 17 00:00:00 2001 From: RyanSpies-NOAA Date: Mon, 27 Nov 2023 17:42:23 +0000 Subject: [PATCH 12/19] Added vrt raster and final mosaic step; fixed logging --- tools/inundate_nation.py | 43 +++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index a25b45feb..a4d258c0a 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -12,8 +12,7 @@ import rasterio from inundate_mosaic_wrapper import produce_mosaicked_inundation -from rasterio.merge import merge - +from osgeo import gdal from utils.shared_functions import FIM_Helpers as fh from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv @@ -43,10 +42,10 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print("Inundation Nation script starting...") fim_version = os.path.basename(os.path.normpath(fim_run_dir)) - logging.info(f"Using fim version: {fim_version}") output_base_file_name = magnitude_key + "_" + fim_version __setup_logger(output_dir, output_base_file_name) + logging.info(f"Using fim version: {fim_version}") start_dt = datetime.now() @@ -72,12 +71,6 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, if huc_list == 'all' or len(huc_list) == 0: huc_list = [] for huc in os.listdir(fim_run_dir): - # if ( - # huc != 'logs' - # and huc != 'branch_errors' - # and huc != 'unit_errors' - # and os.path.isdir(os.path.join(fim_run_dir, huc)) - # ): if re.match(r'\d{8}', huc): huc_list.append(huc) else: @@ -125,6 +118,9 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print(msg) logging.info(msg) + # Perform VRT creation and mosaic all of the huc rasters using boolean rasters + vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name) + # now cleanup the temp bool directory shutil.rmtree(output_bool_dir, ignore_errors=True) @@ -132,7 +128,8 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, print("Skipping mosiaking") # now cleanup the raw mosiac directories - # shutil.rmtree(magnitude_output_dir, ignore_errors=True) + # comment this out if you want to see the individual huc rasters + shutil.rmtree(magnitude_output_dir, ignore_errors=True) fh.print_current_date_time() logging.info(logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))) @@ -213,6 +210,28 @@ def create_bool_rasters(args): ) as dst: dst.write(array.astype(rasterio.int8)) +def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): + + rasters_to_mosaic = [] + for rasfile in os.listdir(output_bool_dir): + if rasfile.endswith('.tif') and "extent" in rasfile: + p = output_bool_dir + os.sep + rasfile + print("Processing: " + p) + rasters_to_mosaic.append(p) + + output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt") + print("Creating virtual raster: " + output_mosiac_vrt) + logging.info("Creating virtual raster: " + output_mosiac_vrt) + vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic) + + output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif") + print("Building raster mosaic: " + output_mosiac_raster) + logging.info("Building raster mosaic: " + output_mosiac_raster) + print("This can take a number of hours, watch 'docker stats' cpu value to ensure the process"\ + "to ensure the process is still working") + gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2']) + vrt = None + def __setup_logger(output_folder_path, log_file_name_key): start_time = datetime.now() @@ -220,8 +239,8 @@ def __setup_logger(output_folder_path, log_file_name_key): log_file_name = f"{log_file_name_key}-{file_dt_string}.log" log_file_path = os.path.join(output_folder_path, log_file_name) - - logging.basicConfig(filename=log_file_path, level=logging.DEBUG, format="%(message)s") + print('Log file created here:' + str(log_file_path)) + logging.basicConfig(filename=log_file_path, level=logging.INFO, format="%(message)s") # yes.. this can do console logs as well, but it can be a bit unstable and ugly From 585208552d0b6423e46ff78e583cede90e4a9add Mon Sep 17 00:00:00 2001 From: RyanSpies-NOAA Date: Mon, 27 Nov 2023 22:54:24 +0000 Subject: [PATCH 13/19] More fixes for logging info and adding log printing to screen --- tools/inundate_nation.py | 56 +++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index a4d258c0a..f546a2f2f 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -55,13 +55,10 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(f"flow_file: {flow_file}") logging.info(f"inc_mosaic: {str(inc_mosaic)}") - print("Preparing to generate inundation outputs for magnitude: " + magnitude_key) - print("Input flow file: " + flow_file) - magnitude_output_dir = os.path.join(output_dir, output_base_file_name) if not os.path.exists(magnitude_output_dir): - print("Creating new output directory for raw mosaic files: " + magnitude_output_dir) + logging.info("Removing previous output dir and creating new output dir for inunation wrapper files: " + magnitude_output_dir) os.mkdir(magnitude_output_dir) else: # we need to empty it. we will kill it and remake it (using rmtree to force it) @@ -80,21 +77,17 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, huc_list.sort() - print("Inundation raw mosaic outputs will saved here: " + magnitude_output_dir) - print() - + logging.info(f"Inundation mosaic wrapper outputs will saved here: {magnitude_output_dir}") run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number]) - print() # Perform mosaic operation if inc_mosaic: fh.print_current_date_time() logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S")) - print("Performing bool mosaic process...") logging.info("Performing bool mosaic process...") output_bool_dir = os.path.join(output_dir, "bool_temp") - print(f"output_bool_dir is {output_bool_dir}") + logging.info(f"output_bool_dir is {output_bool_dir}") if not os.path.exists(output_bool_dir): os.mkdir(output_bool_dir) @@ -158,10 +151,10 @@ def run_inundation(args): inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif") - print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude)) + logging.info("Running inundation wrapper for the NWM recurrence intervals for each huc using magnitude: " + str(magnitude)) print( - "This will take a long time depending on the number of HUCs. No progress bar will be shown." - " Once is gets to mosiacing (if applicable), screen output will exist. To see if the script has frozen," + "This will take a long time depending on the number of HUCs. Progress bar may not appear." + " Once it gets to boolean/mosiacing (if applicable), screen output will exist. To see if the script has frozen," " you should be able to watch the file system for some changes." ) print() @@ -173,7 +166,7 @@ def run_inundation(args): inundation_raster=inundation_raster, num_workers=job_number, remove_intermediate=True, - verbose=False, + verbose=True, is_mosaic_for_branches=True, ) @@ -216,7 +209,6 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): for rasfile in os.listdir(output_bool_dir): if rasfile.endswith('.tif') and "extent" in rasfile: p = output_bool_dir + os.sep + rasfile - print("Processing: " + p) rasters_to_mosaic.append(p) output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt") @@ -225,27 +217,45 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic) output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif") - print("Building raster mosaic: " + output_mosiac_raster) logging.info("Building raster mosaic: " + output_mosiac_raster) - print("This can take a number of hours, watch 'docker stats' cpu value to ensure the process"\ - "to ensure the process is still working") + print("Note: This step can take a number of hours if processing 100s of hucs") gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2']) vrt = None -def __setup_logger(output_folder_path, log_file_name_key): +def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO): start_time = datetime.now() file_dt_string = start_time.strftime("%Y_%m_%d-%H_%M_%S") log_file_name = f"{log_file_name_key}-{file_dt_string}.log" log_file_path = os.path.join(output_folder_path, log_file_name) print('Log file created here:' + str(log_file_path)) - logging.basicConfig(filename=log_file_path, level=logging.INFO, format="%(message)s") - # yes.. this can do console logs as well, but it can be a bit unstable and ugly + # Clear previous logging configuration + logging.getLogger().handlers = [] + + # Create a StreamHandler and set the level + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + + # Create a FileHandler and set the level + file_handler = logging.FileHandler(log_file_path) + file_handler.setLevel(log_level) + + # Create a formatter and set the formatter for the handlers + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + console_handler.setFormatter(formatter) + file_handler.setFormatter(formatter) + + # Add the handlers to the logger + logger = logging.getLogger() + logger.setLevel(log_level) + logger.addHandler(console_handler) + logger.addHandler(file_handler) - logging.info(f'Started : {start_time.strftime("%m/%d/%Y %H:%M:%S")}') - logging.info("----------------") + # Log the start time + logger.info(f'Started: {start_time.strftime("%m/%d/%Y %H:%M:%S")}') + logger.info("----------------") if __name__ == "__main__": From 4452be67cc57bb8ae1d1c89209d696fbb97a2f15 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Mon, 27 Nov 2023 23:27:35 +0000 Subject: [PATCH 14/19] fix linting error --- tools/inundate_nation.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) mode change 100644 => 100755 tools/inundate_nation.py diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py old mode 100644 new mode 100755 index f546a2f2f..e60dc6541 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -13,6 +13,7 @@ import rasterio from inundate_mosaic_wrapper import produce_mosaicked_inundation from osgeo import gdal + from utils.shared_functions import FIM_Helpers as fh from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv @@ -58,7 +59,10 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, magnitude_output_dir = os.path.join(output_dir, output_base_file_name) if not os.path.exists(magnitude_output_dir): - logging.info("Removing previous output dir and creating new output dir for inunation wrapper files: " + magnitude_output_dir) + logging.info( + "Removing previous output dir and creating new output dir for inunation wrapper files: " + + magnitude_output_dir + ) os.mkdir(magnitude_output_dir) else: # we need to empty it. we will kill it and remake it (using rmtree to force it) @@ -80,7 +84,6 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(f"Inundation mosaic wrapper outputs will saved here: {magnitude_output_dir}") run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number]) - # Perform mosaic operation if inc_mosaic: fh.print_current_date_time() @@ -151,7 +154,10 @@ def run_inundation(args): inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif") - logging.info("Running inundation wrapper for the NWM recurrence intervals for each huc using magnitude: " + str(magnitude)) + logging.info( + "Running inundation wrapper for the NWM recurrence intervals for each huc using magnitude: " + + str(magnitude) + ) print( "This will take a long time depending on the number of HUCs. Progress bar may not appear." " Once it gets to boolean/mosiacing (if applicable), screen output will exist. To see if the script has frozen," @@ -203,8 +209,8 @@ def create_bool_rasters(args): ) as dst: dst.write(array.astype(rasterio.int8)) -def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): +def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): rasters_to_mosaic = [] for rasfile in os.listdir(output_bool_dir): if rasfile.endswith('.tif') and "extent" in rasfile: @@ -216,10 +222,16 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): logging.info("Creating virtual raster: " + output_mosiac_vrt) vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic) - output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif") + output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif") logging.info("Building raster mosaic: " + output_mosiac_raster) print("Note: This step can take a number of hours if processing 100s of hucs") - gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2']) + gdal.Translate( + output_mosiac_raster, + vrt, + xRes=10, + yRes=-10, + creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2'], + ) vrt = None From 3445107870f32d288facb07571ed75809a5281e7 Mon Sep 17 00:00:00 2001 From: RyanSpies-NOAA Date: Thu, 30 Nov 2023 16:43:21 +0000 Subject: [PATCH 15/19] Added multi-threading to gdal.translate --- tools/inundate_nation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index f546a2f2f..1eb8d264a 100644 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -112,7 +112,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(msg) # Perform VRT creation and mosaic all of the huc rasters using boolean rasters - vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name) + vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name,job_number) # now cleanup the temp bool directory shutil.rmtree(output_bool_dir, ignore_errors=True) @@ -203,7 +203,7 @@ def create_bool_rasters(args): ) as dst: dst.write(array.astype(rasterio.int8)) -def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): +def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads): rasters_to_mosaic = [] for rasfile in os.listdir(output_bool_dir): @@ -212,14 +212,15 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag): rasters_to_mosaic.append(p) output_mosiac_vrt = os.path.join(output_bool_dir, fim_version_tag + "_merged.vrt") - print("Creating virtual raster: " + output_mosiac_vrt) logging.info("Creating virtual raster: " + output_mosiac_vrt) vrt = gdal.BuildVRT(output_mosiac_vrt, rasters_to_mosaic) output_mosiac_raster = os.path.join(output_dir, fim_version_tag + "_mosaic.tif") logging.info("Building raster mosaic: " + output_mosiac_raster) + logging.info("Using " + str(threads) + " threads for parallizing") print("Note: This step can take a number of hours if processing 100s of hucs") - gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2']) + gdal.Translate(output_mosiac_raster, vrt, xRes = 10, yRes = -10, + creationOptions = ['COMPRESS=LZW','TILED=YES','PREDICTOR=2','NUM_THREADS='+str(threads)]) vrt = None From 15aad0427198d5bece4f33b6b7629f45f1c86417 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Fri, 1 Dec 2023 17:41:26 +0000 Subject: [PATCH 16/19] linting fix --- .gitattributes | 0 .github/ISSUE_TEMPLATE.md | 0 .github/PULL_REQUEST_TEMPLATE.md | 0 .github/workflows/lint_and_format.yaml | 0 .gitignore | 0 .pre-commit-config.yaml | 0 CITATION.cff | 0 CONTRIBUTING.md | 0 Dockerfile | 0 LICENSE | 0 Pipfile | 0 Pipfile.lock | 0 README.md | 0 config/aws_s3_put_fim3_hydrovis_whitelist.lst | 0 config/aws_s3_put_fim4_hydrovis_whitelist.lst | 0 config/deny_branch_unittests.lst | 0 config/deny_branch_zero.lst | 0 config/deny_branches.lst | 0 config/deny_unit.lst | 0 config/params_template.env | 0 config/symbology/esri/agreement_raster.lyr | Bin config/symbology/esri/inundation_raster.lyr | Bin config/symbology/qgis/agreement_raster.qml | 0 config/symbology/qgis/inundation_raster.qml | 0 config/symbology/qgis/inundation_vector.qml | 0 data/README.md | 0 data/__init__.py | 0 data/aws/.gitignore | 0 data/aws/aws_base.py | 0 data/aws/aws_creds_template.env | 0 data/aws/push-hv-data-support-files.sh | 0 data/aws/s3.py | 0 data/bathymetry/preprocess_bathymetry.py | 0 data/ble/ble_benchmark/Dockerfile | 0 data/ble/ble_benchmark/Pipfile.lock | 0 data/ble/ble_benchmark/README.md | 0 data/ble/ble_benchmark/create_ble_benchmark.py | 0 data/create_vrt_file.py | 0 data/esri.py | 0 data/nws/.gitignore | 0 data/usgs/acquire_and_preprocess_3dep_dems.py | 0 data/usgs/preprocess_ahps_usgs.py | 0 data/usgs/preprocess_download_usgs_grids.py | 0 data/write_parquet_from_calib_pts.py | 0 docs/CHANGELOG.md | 0 docs/INSTALL.md | 0 docs/SECURITY.md | 0 docs/TERMS.md | 0 pyproject.toml | 0 src/__init__.py | 0 src/aggregate_by_huc.py | 0 src/aggregate_fim_outputs.py | 0 src/associate_levelpaths_with_levees.py | 0 src/bash_functions.env | 0 src/bash_variables.env | 0 src/bathy_src_adjust_topwidth.py | 0 src/bathymetric_adjustment.py | 0 src/build_stream_traversal.py | 0 src/check_unit_errors.py | 0 src/derive_headwaters.py | 0 src/entrypoint.sh | 0 src/src_adjust_ras2fim_rating.py | 0 src/src_adjust_spatial_obs.py | 0 src/src_adjust_usgs_rating.py | 0 src/src_manual_calibration.py | 0 src/src_roughness_optimization.py | 0 src/utils/__init__.py | 0 src/utils/fim_enums.py | 0 src/utils/shared_functions.py | 0 src/utils/shared_validators.py | 0 src/utils/shared_variables.py | 0 tools/.env.template | 0 tools/.gitignore | 0 tools/adjust_rc_with_feedback.py | 0 tools/aggregate_csv_files.py | 0 tools/check_deep_flooding.py | 0 tools/combine_crosswalk_tables.py | 0 tools/compare_ms_and_non_ms_metrics.py | 0 tools/composite_inundation.py | 0 tools/eval_alt_catfim.py | 0 tools/eval_plots.py | 0 tools/eval_plots_stackedbar.py | 0 tools/evaluate_continuity.py | 0 tools/fimr_to_benchmark.py | 0 tools/find_test_case_folders.py | 0 tools/generate_nws_lid.py | 0 tools/inundate_mosaic_wrapper.py | 0 tools/inundate_nation.py | 6 ++---- tools/overlapping_inundation.py | 0 tools/pixel_counter.py | 0 tools/pixel_counter_functions.py | 0 tools/pixel_counter_wrapper.py | 0 tools/rating_curve_get_usgs_curves.py | 0 tools/test_case_by_hydro_id.py | 0 unit_tests/README.md | 0 unit_tests/__template.json | 0 unit_tests/__template.py | 0 unit_tests/aggregate_branch_lists_params.json | 0 unit_tests/aggregate_branch_lists_test.py | 0 unit_tests/check_unit_errors_params.json | 0 unit_tests/check_unit_errors_test.py | 0 unit_tests/clip_vectors_to_wbd_params.json | 0 unit_tests/clip_vectors_to_wbd_test.py | 0 unit_tests/derive_level_paths_params.json | 0 unit_tests/derive_level_paths_test.py | 0 ...filter_catchments_and_add_attributes_params.json | 0 .../filter_catchments_and_add_attributes_test.py | 0 unit_tests/generate_branch_list_csv_params.json | 0 unit_tests/generate_branch_list_csv_test.py | 0 unit_tests/generate_branch_list_params.json | 0 unit_tests/generate_branch_list_test.py | 0 unit_tests/inundate_gms_params.json | 0 unit_tests/inundate_gms_test.py | 0 unit_tests/inundation_params.json | 0 unit_tests/inundation_test.py | 0 unit_tests/outputs_cleanup_params.json | 0 unit_tests/outputs_cleanup_test.py | 0 unit_tests/pyproject.toml | 0 unit_tests/rating_curve_comparison_params.json | 0 unit_tests/rating_curve_comparison_test.py | 0 unit_tests/shared_functions_params.json | 0 unit_tests/shared_functions_test.py | 0 unit_tests/split_flows_params.json | 0 unit_tests/split_flows_test.py | 0 unit_tests/unit_tests_utils.py | 0 unit_tests/usgs_gage_crosswalk_params.json | 0 unit_tests/usgs_gage_crosswalk_test.py | 0 127 files changed, 2 insertions(+), 4 deletions(-) mode change 100644 => 100755 .gitattributes mode change 100644 => 100755 .github/ISSUE_TEMPLATE.md mode change 100644 => 100755 .github/PULL_REQUEST_TEMPLATE.md mode change 100644 => 100755 .github/workflows/lint_and_format.yaml mode change 100644 => 100755 .gitignore mode change 100644 => 100755 .pre-commit-config.yaml mode change 100644 => 100755 CITATION.cff mode change 100644 => 100755 CONTRIBUTING.md mode change 100644 => 100755 Dockerfile mode change 100644 => 100755 LICENSE mode change 100644 => 100755 Pipfile mode change 100644 => 100755 Pipfile.lock mode change 100644 => 100755 README.md mode change 100644 => 100755 config/aws_s3_put_fim3_hydrovis_whitelist.lst mode change 100644 => 100755 config/aws_s3_put_fim4_hydrovis_whitelist.lst mode change 100644 => 100755 config/deny_branch_unittests.lst mode change 100644 => 100755 config/deny_branch_zero.lst mode change 100644 => 100755 config/deny_branches.lst mode change 100644 => 100755 config/deny_unit.lst mode change 100644 => 100755 config/params_template.env mode change 100644 => 100755 config/symbology/esri/agreement_raster.lyr mode change 100644 => 100755 config/symbology/esri/inundation_raster.lyr mode change 100644 => 100755 config/symbology/qgis/agreement_raster.qml mode change 100644 => 100755 config/symbology/qgis/inundation_raster.qml mode change 100644 => 100755 config/symbology/qgis/inundation_vector.qml mode change 100644 => 100755 data/README.md mode change 100644 => 100755 data/__init__.py mode change 100644 => 100755 data/aws/.gitignore mode change 100644 => 100755 data/aws/aws_base.py mode change 100644 => 100755 data/aws/aws_creds_template.env mode change 100644 => 100755 data/aws/push-hv-data-support-files.sh mode change 100644 => 100755 data/aws/s3.py mode change 100644 => 100755 data/bathymetry/preprocess_bathymetry.py mode change 100644 => 100755 data/ble/ble_benchmark/Dockerfile mode change 100644 => 100755 data/ble/ble_benchmark/Pipfile.lock mode change 100644 => 100755 data/ble/ble_benchmark/README.md mode change 100644 => 100755 data/ble/ble_benchmark/create_ble_benchmark.py mode change 100644 => 100755 data/create_vrt_file.py mode change 100644 => 100755 data/esri.py mode change 100644 => 100755 data/nws/.gitignore mode change 100644 => 100755 data/usgs/acquire_and_preprocess_3dep_dems.py mode change 100644 => 100755 data/usgs/preprocess_ahps_usgs.py mode change 100644 => 100755 data/usgs/preprocess_download_usgs_grids.py mode change 100644 => 100755 data/write_parquet_from_calib_pts.py mode change 100644 => 100755 docs/CHANGELOG.md mode change 100644 => 100755 docs/INSTALL.md mode change 100644 => 100755 docs/SECURITY.md mode change 100644 => 100755 docs/TERMS.md mode change 100644 => 100755 pyproject.toml mode change 100644 => 100755 src/__init__.py mode change 100644 => 100755 src/aggregate_by_huc.py mode change 100644 => 100755 src/aggregate_fim_outputs.py mode change 100644 => 100755 src/associate_levelpaths_with_levees.py mode change 100644 => 100755 src/bash_functions.env mode change 100644 => 100755 src/bash_variables.env mode change 100644 => 100755 src/bathy_src_adjust_topwidth.py mode change 100644 => 100755 src/bathymetric_adjustment.py mode change 100644 => 100755 src/build_stream_traversal.py mode change 100644 => 100755 src/check_unit_errors.py mode change 100644 => 100755 src/derive_headwaters.py mode change 100644 => 100755 src/entrypoint.sh mode change 100644 => 100755 src/src_adjust_ras2fim_rating.py mode change 100644 => 100755 src/src_adjust_spatial_obs.py mode change 100644 => 100755 src/src_adjust_usgs_rating.py mode change 100644 => 100755 src/src_manual_calibration.py mode change 100644 => 100755 src/src_roughness_optimization.py mode change 100644 => 100755 src/utils/__init__.py mode change 100644 => 100755 src/utils/fim_enums.py mode change 100644 => 100755 src/utils/shared_functions.py mode change 100644 => 100755 src/utils/shared_validators.py mode change 100644 => 100755 src/utils/shared_variables.py mode change 100644 => 100755 tools/.env.template mode change 100644 => 100755 tools/.gitignore mode change 100644 => 100755 tools/adjust_rc_with_feedback.py mode change 100644 => 100755 tools/aggregate_csv_files.py mode change 100644 => 100755 tools/check_deep_flooding.py mode change 100644 => 100755 tools/combine_crosswalk_tables.py mode change 100644 => 100755 tools/compare_ms_and_non_ms_metrics.py mode change 100644 => 100755 tools/composite_inundation.py mode change 100644 => 100755 tools/eval_alt_catfim.py mode change 100644 => 100755 tools/eval_plots.py mode change 100644 => 100755 tools/eval_plots_stackedbar.py mode change 100644 => 100755 tools/evaluate_continuity.py mode change 100644 => 100755 tools/fimr_to_benchmark.py mode change 100644 => 100755 tools/find_test_case_folders.py mode change 100644 => 100755 tools/generate_nws_lid.py mode change 100644 => 100755 tools/inundate_mosaic_wrapper.py mode change 100644 => 100755 tools/overlapping_inundation.py mode change 100644 => 100755 tools/pixel_counter.py mode change 100644 => 100755 tools/pixel_counter_functions.py mode change 100644 => 100755 tools/pixel_counter_wrapper.py mode change 100644 => 100755 tools/rating_curve_get_usgs_curves.py mode change 100644 => 100755 tools/test_case_by_hydro_id.py mode change 100644 => 100755 unit_tests/README.md mode change 100644 => 100755 unit_tests/__template.json mode change 100644 => 100755 unit_tests/__template.py mode change 100644 => 100755 unit_tests/aggregate_branch_lists_params.json mode change 100644 => 100755 unit_tests/aggregate_branch_lists_test.py mode change 100644 => 100755 unit_tests/check_unit_errors_params.json mode change 100644 => 100755 unit_tests/check_unit_errors_test.py mode change 100644 => 100755 unit_tests/clip_vectors_to_wbd_params.json mode change 100644 => 100755 unit_tests/clip_vectors_to_wbd_test.py mode change 100644 => 100755 unit_tests/derive_level_paths_params.json mode change 100644 => 100755 unit_tests/derive_level_paths_test.py mode change 100644 => 100755 unit_tests/filter_catchments_and_add_attributes_params.json mode change 100644 => 100755 unit_tests/filter_catchments_and_add_attributes_test.py mode change 100644 => 100755 unit_tests/generate_branch_list_csv_params.json mode change 100644 => 100755 unit_tests/generate_branch_list_csv_test.py mode change 100644 => 100755 unit_tests/generate_branch_list_params.json mode change 100644 => 100755 unit_tests/generate_branch_list_test.py mode change 100644 => 100755 unit_tests/inundate_gms_params.json mode change 100644 => 100755 unit_tests/inundate_gms_test.py mode change 100644 => 100755 unit_tests/inundation_params.json mode change 100644 => 100755 unit_tests/inundation_test.py mode change 100644 => 100755 unit_tests/outputs_cleanup_params.json mode change 100644 => 100755 unit_tests/outputs_cleanup_test.py mode change 100644 => 100755 unit_tests/pyproject.toml mode change 100644 => 100755 unit_tests/rating_curve_comparison_params.json mode change 100644 => 100755 unit_tests/rating_curve_comparison_test.py mode change 100644 => 100755 unit_tests/shared_functions_params.json mode change 100644 => 100755 unit_tests/shared_functions_test.py mode change 100644 => 100755 unit_tests/split_flows_params.json mode change 100644 => 100755 unit_tests/split_flows_test.py mode change 100644 => 100755 unit_tests/unit_tests_utils.py mode change 100644 => 100755 unit_tests/usgs_gage_crosswalk_params.json mode change 100644 => 100755 unit_tests/usgs_gage_crosswalk_test.py diff --git a/.gitattributes b/.gitattributes old mode 100644 new mode 100755 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md old mode 100644 new mode 100755 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md old mode 100644 new mode 100755 diff --git a/.github/workflows/lint_and_format.yaml b/.github/workflows/lint_and_format.yaml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml old mode 100644 new mode 100755 diff --git a/CITATION.cff b/CITATION.cff old mode 100644 new mode 100755 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md old mode 100644 new mode 100755 diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/Pipfile b/Pipfile old mode 100644 new mode 100755 diff --git a/Pipfile.lock b/Pipfile.lock old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/config/aws_s3_put_fim3_hydrovis_whitelist.lst b/config/aws_s3_put_fim3_hydrovis_whitelist.lst old mode 100644 new mode 100755 diff --git a/config/aws_s3_put_fim4_hydrovis_whitelist.lst b/config/aws_s3_put_fim4_hydrovis_whitelist.lst old mode 100644 new mode 100755 diff --git a/config/deny_branch_unittests.lst b/config/deny_branch_unittests.lst old mode 100644 new mode 100755 diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst old mode 100644 new mode 100755 diff --git a/config/deny_branches.lst b/config/deny_branches.lst old mode 100644 new mode 100755 diff --git a/config/deny_unit.lst b/config/deny_unit.lst old mode 100644 new mode 100755 diff --git a/config/params_template.env b/config/params_template.env old mode 100644 new mode 100755 diff --git a/config/symbology/esri/agreement_raster.lyr b/config/symbology/esri/agreement_raster.lyr old mode 100644 new mode 100755 diff --git a/config/symbology/esri/inundation_raster.lyr b/config/symbology/esri/inundation_raster.lyr old mode 100644 new mode 100755 diff --git a/config/symbology/qgis/agreement_raster.qml b/config/symbology/qgis/agreement_raster.qml old mode 100644 new mode 100755 diff --git a/config/symbology/qgis/inundation_raster.qml b/config/symbology/qgis/inundation_raster.qml old mode 100644 new mode 100755 diff --git a/config/symbology/qgis/inundation_vector.qml b/config/symbology/qgis/inundation_vector.qml old mode 100644 new mode 100755 diff --git a/data/README.md b/data/README.md old mode 100644 new mode 100755 diff --git a/data/__init__.py b/data/__init__.py old mode 100644 new mode 100755 diff --git a/data/aws/.gitignore b/data/aws/.gitignore old mode 100644 new mode 100755 diff --git a/data/aws/aws_base.py b/data/aws/aws_base.py old mode 100644 new mode 100755 diff --git a/data/aws/aws_creds_template.env b/data/aws/aws_creds_template.env old mode 100644 new mode 100755 diff --git a/data/aws/push-hv-data-support-files.sh b/data/aws/push-hv-data-support-files.sh old mode 100644 new mode 100755 diff --git a/data/aws/s3.py b/data/aws/s3.py old mode 100644 new mode 100755 diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py old mode 100644 new mode 100755 diff --git a/data/ble/ble_benchmark/Dockerfile b/data/ble/ble_benchmark/Dockerfile old mode 100644 new mode 100755 diff --git a/data/ble/ble_benchmark/Pipfile.lock b/data/ble/ble_benchmark/Pipfile.lock old mode 100644 new mode 100755 diff --git a/data/ble/ble_benchmark/README.md b/data/ble/ble_benchmark/README.md old mode 100644 new mode 100755 diff --git a/data/ble/ble_benchmark/create_ble_benchmark.py b/data/ble/ble_benchmark/create_ble_benchmark.py old mode 100644 new mode 100755 diff --git a/data/create_vrt_file.py b/data/create_vrt_file.py old mode 100644 new mode 100755 diff --git a/data/esri.py b/data/esri.py old mode 100644 new mode 100755 diff --git a/data/nws/.gitignore b/data/nws/.gitignore old mode 100644 new mode 100755 diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py old mode 100644 new mode 100755 diff --git a/data/usgs/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py old mode 100644 new mode 100755 diff --git a/data/usgs/preprocess_download_usgs_grids.py b/data/usgs/preprocess_download_usgs_grids.py old mode 100644 new mode 100755 diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py old mode 100644 new mode 100755 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md old mode 100644 new mode 100755 diff --git a/docs/INSTALL.md b/docs/INSTALL.md old mode 100644 new mode 100755 diff --git a/docs/SECURITY.md b/docs/SECURITY.md old mode 100644 new mode 100755 diff --git a/docs/TERMS.md b/docs/TERMS.md old mode 100644 new mode 100755 diff --git a/pyproject.toml b/pyproject.toml old mode 100644 new mode 100755 diff --git a/src/__init__.py b/src/__init__.py old mode 100644 new mode 100755 diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py old mode 100644 new mode 100755 diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py old mode 100644 new mode 100755 diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py old mode 100644 new mode 100755 diff --git a/src/bash_functions.env b/src/bash_functions.env old mode 100644 new mode 100755 diff --git a/src/bash_variables.env b/src/bash_variables.env old mode 100644 new mode 100755 diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py old mode 100644 new mode 100755 diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py old mode 100644 new mode 100755 diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py old mode 100644 new mode 100755 diff --git a/src/check_unit_errors.py b/src/check_unit_errors.py old mode 100644 new mode 100755 diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py old mode 100644 new mode 100755 diff --git a/src/entrypoint.sh b/src/entrypoint.sh old mode 100644 new mode 100755 diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py old mode 100644 new mode 100755 diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py old mode 100644 new mode 100755 diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py old mode 100644 new mode 100755 diff --git a/src/src_manual_calibration.py b/src/src_manual_calibration.py old mode 100644 new mode 100755 diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py old mode 100644 new mode 100755 diff --git a/src/utils/__init__.py b/src/utils/__init__.py old mode 100644 new mode 100755 diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py old mode 100644 new mode 100755 diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py old mode 100644 new mode 100755 diff --git a/src/utils/shared_validators.py b/src/utils/shared_validators.py old mode 100644 new mode 100755 diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py old mode 100644 new mode 100755 diff --git a/tools/.env.template b/tools/.env.template old mode 100644 new mode 100755 diff --git a/tools/.gitignore b/tools/.gitignore old mode 100644 new mode 100755 diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py old mode 100644 new mode 100755 diff --git a/tools/aggregate_csv_files.py b/tools/aggregate_csv_files.py old mode 100644 new mode 100755 diff --git a/tools/check_deep_flooding.py b/tools/check_deep_flooding.py old mode 100644 new mode 100755 diff --git a/tools/combine_crosswalk_tables.py b/tools/combine_crosswalk_tables.py old mode 100644 new mode 100755 diff --git a/tools/compare_ms_and_non_ms_metrics.py b/tools/compare_ms_and_non_ms_metrics.py old mode 100644 new mode 100755 diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py old mode 100644 new mode 100755 diff --git a/tools/eval_alt_catfim.py b/tools/eval_alt_catfim.py old mode 100644 new mode 100755 diff --git a/tools/eval_plots.py b/tools/eval_plots.py old mode 100644 new mode 100755 diff --git a/tools/eval_plots_stackedbar.py b/tools/eval_plots_stackedbar.py old mode 100644 new mode 100755 diff --git a/tools/evaluate_continuity.py b/tools/evaluate_continuity.py old mode 100644 new mode 100755 diff --git a/tools/fimr_to_benchmark.py b/tools/fimr_to_benchmark.py old mode 100644 new mode 100755 diff --git a/tools/find_test_case_folders.py b/tools/find_test_case_folders.py old mode 100644 new mode 100755 diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py old mode 100644 new mode 100755 diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py old mode 100644 new mode 100755 diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index d929cf2ad..050436670 100755 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -6,7 +6,6 @@ import os import re import shutil -import sys from datetime import datetime from multiprocessing import Pool @@ -15,7 +14,6 @@ from osgeo import gdal from utils.shared_functions import FIM_Helpers as fh -from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv # INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/' @@ -115,7 +113,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(msg) # Perform VRT creation and mosaic all of the huc rasters using boolean rasters - vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name,job_number) + vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name, job_number) # now cleanup the temp bool directory shutil.rmtree(output_bool_dir, ignore_errors=True) @@ -230,7 +228,7 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads): vrt, xRes=10, yRes=-10, - creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2','NUM_THREADS='+str(threads)], + creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS=' + str(threads)], ) vrt = None diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py old mode 100644 new mode 100755 diff --git a/tools/pixel_counter.py b/tools/pixel_counter.py old mode 100644 new mode 100755 diff --git a/tools/pixel_counter_functions.py b/tools/pixel_counter_functions.py old mode 100644 new mode 100755 diff --git a/tools/pixel_counter_wrapper.py b/tools/pixel_counter_wrapper.py old mode 100644 new mode 100755 diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py old mode 100644 new mode 100755 diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py old mode 100644 new mode 100755 diff --git a/unit_tests/README.md b/unit_tests/README.md old mode 100644 new mode 100755 diff --git a/unit_tests/__template.json b/unit_tests/__template.json old mode 100644 new mode 100755 diff --git a/unit_tests/__template.py b/unit_tests/__template.py old mode 100644 new mode 100755 diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/aggregate_branch_lists_test.py b/unit_tests/aggregate_branch_lists_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/filter_catchments_and_add_attributes_test.py b/unit_tests/filter_catchments_and_add_attributes_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/generate_branch_list_csv_test.py b/unit_tests/generate_branch_list_csv_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/generate_branch_list_test.py b/unit_tests/generate_branch_list_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/inundate_gms_params.json b/unit_tests/inundate_gms_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/inundate_gms_test.py b/unit_tests/inundate_gms_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/inundation_params.json b/unit_tests/inundation_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/inundation_test.py b/unit_tests/inundation_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/pyproject.toml b/unit_tests/pyproject.toml old mode 100644 new mode 100755 diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/shared_functions_test.py b/unit_tests/shared_functions_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py old mode 100644 new mode 100755 diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py old mode 100644 new mode 100755 diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json old mode 100644 new mode 100755 diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py old mode 100644 new mode 100755 From e445a635e35caceabe3928be3133d22573e87ff5 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Fri, 1 Dec 2023 17:43:18 +0000 Subject: [PATCH 17/19] Revert "linting fix" This reverts commit 15aad0427198d5bece4f33b6b7629f45f1c86417. --- .gitattributes | 0 .github/ISSUE_TEMPLATE.md | 0 .github/PULL_REQUEST_TEMPLATE.md | 0 .github/workflows/lint_and_format.yaml | 0 .gitignore | 0 .pre-commit-config.yaml | 0 CITATION.cff | 0 CONTRIBUTING.md | 0 Dockerfile | 0 LICENSE | 0 Pipfile | 0 Pipfile.lock | 0 README.md | 0 config/aws_s3_put_fim3_hydrovis_whitelist.lst | 0 config/aws_s3_put_fim4_hydrovis_whitelist.lst | 0 config/deny_branch_unittests.lst | 0 config/deny_branch_zero.lst | 0 config/deny_branches.lst | 0 config/deny_unit.lst | 0 config/params_template.env | 0 config/symbology/esri/agreement_raster.lyr | Bin config/symbology/esri/inundation_raster.lyr | Bin config/symbology/qgis/agreement_raster.qml | 0 config/symbology/qgis/inundation_raster.qml | 0 config/symbology/qgis/inundation_vector.qml | 0 data/README.md | 0 data/__init__.py | 0 data/aws/.gitignore | 0 data/aws/aws_base.py | 0 data/aws/aws_creds_template.env | 0 data/aws/push-hv-data-support-files.sh | 0 data/aws/s3.py | 0 data/bathymetry/preprocess_bathymetry.py | 0 data/ble/ble_benchmark/Dockerfile | 0 data/ble/ble_benchmark/Pipfile.lock | 0 data/ble/ble_benchmark/README.md | 0 data/ble/ble_benchmark/create_ble_benchmark.py | 0 data/create_vrt_file.py | 0 data/esri.py | 0 data/nws/.gitignore | 0 data/usgs/acquire_and_preprocess_3dep_dems.py | 0 data/usgs/preprocess_ahps_usgs.py | 0 data/usgs/preprocess_download_usgs_grids.py | 0 data/write_parquet_from_calib_pts.py | 0 docs/CHANGELOG.md | 0 docs/INSTALL.md | 0 docs/SECURITY.md | 0 docs/TERMS.md | 0 pyproject.toml | 0 src/__init__.py | 0 src/aggregate_by_huc.py | 0 src/aggregate_fim_outputs.py | 0 src/associate_levelpaths_with_levees.py | 0 src/bash_functions.env | 0 src/bash_variables.env | 0 src/bathy_src_adjust_topwidth.py | 0 src/bathymetric_adjustment.py | 0 src/build_stream_traversal.py | 0 src/check_unit_errors.py | 0 src/derive_headwaters.py | 0 src/entrypoint.sh | 0 src/src_adjust_ras2fim_rating.py | 0 src/src_adjust_spatial_obs.py | 0 src/src_adjust_usgs_rating.py | 0 src/src_manual_calibration.py | 0 src/src_roughness_optimization.py | 0 src/utils/__init__.py | 0 src/utils/fim_enums.py | 0 src/utils/shared_functions.py | 0 src/utils/shared_validators.py | 0 src/utils/shared_variables.py | 0 tools/.env.template | 0 tools/.gitignore | 0 tools/adjust_rc_with_feedback.py | 0 tools/aggregate_csv_files.py | 0 tools/check_deep_flooding.py | 0 tools/combine_crosswalk_tables.py | 0 tools/compare_ms_and_non_ms_metrics.py | 0 tools/composite_inundation.py | 0 tools/eval_alt_catfim.py | 0 tools/eval_plots.py | 0 tools/eval_plots_stackedbar.py | 0 tools/evaluate_continuity.py | 0 tools/fimr_to_benchmark.py | 0 tools/find_test_case_folders.py | 0 tools/generate_nws_lid.py | 0 tools/inundate_mosaic_wrapper.py | 0 tools/inundate_nation.py | 6 ++++-- tools/overlapping_inundation.py | 0 tools/pixel_counter.py | 0 tools/pixel_counter_functions.py | 0 tools/pixel_counter_wrapper.py | 0 tools/rating_curve_get_usgs_curves.py | 0 tools/test_case_by_hydro_id.py | 0 unit_tests/README.md | 0 unit_tests/__template.json | 0 unit_tests/__template.py | 0 unit_tests/aggregate_branch_lists_params.json | 0 unit_tests/aggregate_branch_lists_test.py | 0 unit_tests/check_unit_errors_params.json | 0 unit_tests/check_unit_errors_test.py | 0 unit_tests/clip_vectors_to_wbd_params.json | 0 unit_tests/clip_vectors_to_wbd_test.py | 0 unit_tests/derive_level_paths_params.json | 0 unit_tests/derive_level_paths_test.py | 0 ...filter_catchments_and_add_attributes_params.json | 0 .../filter_catchments_and_add_attributes_test.py | 0 unit_tests/generate_branch_list_csv_params.json | 0 unit_tests/generate_branch_list_csv_test.py | 0 unit_tests/generate_branch_list_params.json | 0 unit_tests/generate_branch_list_test.py | 0 unit_tests/inundate_gms_params.json | 0 unit_tests/inundate_gms_test.py | 0 unit_tests/inundation_params.json | 0 unit_tests/inundation_test.py | 0 unit_tests/outputs_cleanup_params.json | 0 unit_tests/outputs_cleanup_test.py | 0 unit_tests/pyproject.toml | 0 unit_tests/rating_curve_comparison_params.json | 0 unit_tests/rating_curve_comparison_test.py | 0 unit_tests/shared_functions_params.json | 0 unit_tests/shared_functions_test.py | 0 unit_tests/split_flows_params.json | 0 unit_tests/split_flows_test.py | 0 unit_tests/unit_tests_utils.py | 0 unit_tests/usgs_gage_crosswalk_params.json | 0 unit_tests/usgs_gage_crosswalk_test.py | 0 127 files changed, 4 insertions(+), 2 deletions(-) mode change 100755 => 100644 .gitattributes mode change 100755 => 100644 .github/ISSUE_TEMPLATE.md mode change 100755 => 100644 .github/PULL_REQUEST_TEMPLATE.md mode change 100755 => 100644 .github/workflows/lint_and_format.yaml mode change 100755 => 100644 .gitignore mode change 100755 => 100644 .pre-commit-config.yaml mode change 100755 => 100644 CITATION.cff mode change 100755 => 100644 CONTRIBUTING.md mode change 100755 => 100644 Dockerfile mode change 100755 => 100644 LICENSE mode change 100755 => 100644 Pipfile mode change 100755 => 100644 Pipfile.lock mode change 100755 => 100644 README.md mode change 100755 => 100644 config/aws_s3_put_fim3_hydrovis_whitelist.lst mode change 100755 => 100644 config/aws_s3_put_fim4_hydrovis_whitelist.lst mode change 100755 => 100644 config/deny_branch_unittests.lst mode change 100755 => 100644 config/deny_branch_zero.lst mode change 100755 => 100644 config/deny_branches.lst mode change 100755 => 100644 config/deny_unit.lst mode change 100755 => 100644 config/params_template.env mode change 100755 => 100644 config/symbology/esri/agreement_raster.lyr mode change 100755 => 100644 config/symbology/esri/inundation_raster.lyr mode change 100755 => 100644 config/symbology/qgis/agreement_raster.qml mode change 100755 => 100644 config/symbology/qgis/inundation_raster.qml mode change 100755 => 100644 config/symbology/qgis/inundation_vector.qml mode change 100755 => 100644 data/README.md mode change 100755 => 100644 data/__init__.py mode change 100755 => 100644 data/aws/.gitignore mode change 100755 => 100644 data/aws/aws_base.py mode change 100755 => 100644 data/aws/aws_creds_template.env mode change 100755 => 100644 data/aws/push-hv-data-support-files.sh mode change 100755 => 100644 data/aws/s3.py mode change 100755 => 100644 data/bathymetry/preprocess_bathymetry.py mode change 100755 => 100644 data/ble/ble_benchmark/Dockerfile mode change 100755 => 100644 data/ble/ble_benchmark/Pipfile.lock mode change 100755 => 100644 data/ble/ble_benchmark/README.md mode change 100755 => 100644 data/ble/ble_benchmark/create_ble_benchmark.py mode change 100755 => 100644 data/create_vrt_file.py mode change 100755 => 100644 data/esri.py mode change 100755 => 100644 data/nws/.gitignore mode change 100755 => 100644 data/usgs/acquire_and_preprocess_3dep_dems.py mode change 100755 => 100644 data/usgs/preprocess_ahps_usgs.py mode change 100755 => 100644 data/usgs/preprocess_download_usgs_grids.py mode change 100755 => 100644 data/write_parquet_from_calib_pts.py mode change 100755 => 100644 docs/CHANGELOG.md mode change 100755 => 100644 docs/INSTALL.md mode change 100755 => 100644 docs/SECURITY.md mode change 100755 => 100644 docs/TERMS.md mode change 100755 => 100644 pyproject.toml mode change 100755 => 100644 src/__init__.py mode change 100755 => 100644 src/aggregate_by_huc.py mode change 100755 => 100644 src/aggregate_fim_outputs.py mode change 100755 => 100644 src/associate_levelpaths_with_levees.py mode change 100755 => 100644 src/bash_functions.env mode change 100755 => 100644 src/bash_variables.env mode change 100755 => 100644 src/bathy_src_adjust_topwidth.py mode change 100755 => 100644 src/bathymetric_adjustment.py mode change 100755 => 100644 src/build_stream_traversal.py mode change 100755 => 100644 src/check_unit_errors.py mode change 100755 => 100644 src/derive_headwaters.py mode change 100755 => 100644 src/entrypoint.sh mode change 100755 => 100644 src/src_adjust_ras2fim_rating.py mode change 100755 => 100644 src/src_adjust_spatial_obs.py mode change 100755 => 100644 src/src_adjust_usgs_rating.py mode change 100755 => 100644 src/src_manual_calibration.py mode change 100755 => 100644 src/src_roughness_optimization.py mode change 100755 => 100644 src/utils/__init__.py mode change 100755 => 100644 src/utils/fim_enums.py mode change 100755 => 100644 src/utils/shared_functions.py mode change 100755 => 100644 src/utils/shared_validators.py mode change 100755 => 100644 src/utils/shared_variables.py mode change 100755 => 100644 tools/.env.template mode change 100755 => 100644 tools/.gitignore mode change 100755 => 100644 tools/adjust_rc_with_feedback.py mode change 100755 => 100644 tools/aggregate_csv_files.py mode change 100755 => 100644 tools/check_deep_flooding.py mode change 100755 => 100644 tools/combine_crosswalk_tables.py mode change 100755 => 100644 tools/compare_ms_and_non_ms_metrics.py mode change 100755 => 100644 tools/composite_inundation.py mode change 100755 => 100644 tools/eval_alt_catfim.py mode change 100755 => 100644 tools/eval_plots.py mode change 100755 => 100644 tools/eval_plots_stackedbar.py mode change 100755 => 100644 tools/evaluate_continuity.py mode change 100755 => 100644 tools/fimr_to_benchmark.py mode change 100755 => 100644 tools/find_test_case_folders.py mode change 100755 => 100644 tools/generate_nws_lid.py mode change 100755 => 100644 tools/inundate_mosaic_wrapper.py mode change 100755 => 100644 tools/overlapping_inundation.py mode change 100755 => 100644 tools/pixel_counter.py mode change 100755 => 100644 tools/pixel_counter_functions.py mode change 100755 => 100644 tools/pixel_counter_wrapper.py mode change 100755 => 100644 tools/rating_curve_get_usgs_curves.py mode change 100755 => 100644 tools/test_case_by_hydro_id.py mode change 100755 => 100644 unit_tests/README.md mode change 100755 => 100644 unit_tests/__template.json mode change 100755 => 100644 unit_tests/__template.py mode change 100755 => 100644 unit_tests/aggregate_branch_lists_params.json mode change 100755 => 100644 unit_tests/aggregate_branch_lists_test.py mode change 100755 => 100644 unit_tests/check_unit_errors_params.json mode change 100755 => 100644 unit_tests/check_unit_errors_test.py mode change 100755 => 100644 unit_tests/clip_vectors_to_wbd_params.json mode change 100755 => 100644 unit_tests/clip_vectors_to_wbd_test.py mode change 100755 => 100644 unit_tests/derive_level_paths_params.json mode change 100755 => 100644 unit_tests/derive_level_paths_test.py mode change 100755 => 100644 unit_tests/filter_catchments_and_add_attributes_params.json mode change 100755 => 100644 unit_tests/filter_catchments_and_add_attributes_test.py mode change 100755 => 100644 unit_tests/generate_branch_list_csv_params.json mode change 100755 => 100644 unit_tests/generate_branch_list_csv_test.py mode change 100755 => 100644 unit_tests/generate_branch_list_params.json mode change 100755 => 100644 unit_tests/generate_branch_list_test.py mode change 100755 => 100644 unit_tests/inundate_gms_params.json mode change 100755 => 100644 unit_tests/inundate_gms_test.py mode change 100755 => 100644 unit_tests/inundation_params.json mode change 100755 => 100644 unit_tests/inundation_test.py mode change 100755 => 100644 unit_tests/outputs_cleanup_params.json mode change 100755 => 100644 unit_tests/outputs_cleanup_test.py mode change 100755 => 100644 unit_tests/pyproject.toml mode change 100755 => 100644 unit_tests/rating_curve_comparison_params.json mode change 100755 => 100644 unit_tests/rating_curve_comparison_test.py mode change 100755 => 100644 unit_tests/shared_functions_params.json mode change 100755 => 100644 unit_tests/shared_functions_test.py mode change 100755 => 100644 unit_tests/split_flows_params.json mode change 100755 => 100644 unit_tests/split_flows_test.py mode change 100755 => 100644 unit_tests/unit_tests_utils.py mode change 100755 => 100644 unit_tests/usgs_gage_crosswalk_params.json mode change 100755 => 100644 unit_tests/usgs_gage_crosswalk_test.py diff --git a/.gitattributes b/.gitattributes old mode 100755 new mode 100644 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md old mode 100755 new mode 100644 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md old mode 100755 new mode 100644 diff --git a/.github/workflows/lint_and_format.yaml b/.github/workflows/lint_and_format.yaml old mode 100755 new mode 100644 diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml old mode 100755 new mode 100644 diff --git a/CITATION.cff b/CITATION.cff old mode 100755 new mode 100644 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md old mode 100755 new mode 100644 diff --git a/Dockerfile b/Dockerfile old mode 100755 new mode 100644 diff --git a/LICENSE b/LICENSE old mode 100755 new mode 100644 diff --git a/Pipfile b/Pipfile old mode 100755 new mode 100644 diff --git a/Pipfile.lock b/Pipfile.lock old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 diff --git a/config/aws_s3_put_fim3_hydrovis_whitelist.lst b/config/aws_s3_put_fim3_hydrovis_whitelist.lst old mode 100755 new mode 100644 diff --git a/config/aws_s3_put_fim4_hydrovis_whitelist.lst b/config/aws_s3_put_fim4_hydrovis_whitelist.lst old mode 100755 new mode 100644 diff --git a/config/deny_branch_unittests.lst b/config/deny_branch_unittests.lst old mode 100755 new mode 100644 diff --git a/config/deny_branch_zero.lst b/config/deny_branch_zero.lst old mode 100755 new mode 100644 diff --git a/config/deny_branches.lst b/config/deny_branches.lst old mode 100755 new mode 100644 diff --git a/config/deny_unit.lst b/config/deny_unit.lst old mode 100755 new mode 100644 diff --git a/config/params_template.env b/config/params_template.env old mode 100755 new mode 100644 diff --git a/config/symbology/esri/agreement_raster.lyr b/config/symbology/esri/agreement_raster.lyr old mode 100755 new mode 100644 diff --git a/config/symbology/esri/inundation_raster.lyr b/config/symbology/esri/inundation_raster.lyr old mode 100755 new mode 100644 diff --git a/config/symbology/qgis/agreement_raster.qml b/config/symbology/qgis/agreement_raster.qml old mode 100755 new mode 100644 diff --git a/config/symbology/qgis/inundation_raster.qml b/config/symbology/qgis/inundation_raster.qml old mode 100755 new mode 100644 diff --git a/config/symbology/qgis/inundation_vector.qml b/config/symbology/qgis/inundation_vector.qml old mode 100755 new mode 100644 diff --git a/data/README.md b/data/README.md old mode 100755 new mode 100644 diff --git a/data/__init__.py b/data/__init__.py old mode 100755 new mode 100644 diff --git a/data/aws/.gitignore b/data/aws/.gitignore old mode 100755 new mode 100644 diff --git a/data/aws/aws_base.py b/data/aws/aws_base.py old mode 100755 new mode 100644 diff --git a/data/aws/aws_creds_template.env b/data/aws/aws_creds_template.env old mode 100755 new mode 100644 diff --git a/data/aws/push-hv-data-support-files.sh b/data/aws/push-hv-data-support-files.sh old mode 100755 new mode 100644 diff --git a/data/aws/s3.py b/data/aws/s3.py old mode 100755 new mode 100644 diff --git a/data/bathymetry/preprocess_bathymetry.py b/data/bathymetry/preprocess_bathymetry.py old mode 100755 new mode 100644 diff --git a/data/ble/ble_benchmark/Dockerfile b/data/ble/ble_benchmark/Dockerfile old mode 100755 new mode 100644 diff --git a/data/ble/ble_benchmark/Pipfile.lock b/data/ble/ble_benchmark/Pipfile.lock old mode 100755 new mode 100644 diff --git a/data/ble/ble_benchmark/README.md b/data/ble/ble_benchmark/README.md old mode 100755 new mode 100644 diff --git a/data/ble/ble_benchmark/create_ble_benchmark.py b/data/ble/ble_benchmark/create_ble_benchmark.py old mode 100755 new mode 100644 diff --git a/data/create_vrt_file.py b/data/create_vrt_file.py old mode 100755 new mode 100644 diff --git a/data/esri.py b/data/esri.py old mode 100755 new mode 100644 diff --git a/data/nws/.gitignore b/data/nws/.gitignore old mode 100755 new mode 100644 diff --git a/data/usgs/acquire_and_preprocess_3dep_dems.py b/data/usgs/acquire_and_preprocess_3dep_dems.py old mode 100755 new mode 100644 diff --git a/data/usgs/preprocess_ahps_usgs.py b/data/usgs/preprocess_ahps_usgs.py old mode 100755 new mode 100644 diff --git a/data/usgs/preprocess_download_usgs_grids.py b/data/usgs/preprocess_download_usgs_grids.py old mode 100755 new mode 100644 diff --git a/data/write_parquet_from_calib_pts.py b/data/write_parquet_from_calib_pts.py old mode 100755 new mode 100644 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md old mode 100755 new mode 100644 diff --git a/docs/INSTALL.md b/docs/INSTALL.md old mode 100755 new mode 100644 diff --git a/docs/SECURITY.md b/docs/SECURITY.md old mode 100755 new mode 100644 diff --git a/docs/TERMS.md b/docs/TERMS.md old mode 100755 new mode 100644 diff --git a/pyproject.toml b/pyproject.toml old mode 100755 new mode 100644 diff --git a/src/__init__.py b/src/__init__.py old mode 100755 new mode 100644 diff --git a/src/aggregate_by_huc.py b/src/aggregate_by_huc.py old mode 100755 new mode 100644 diff --git a/src/aggregate_fim_outputs.py b/src/aggregate_fim_outputs.py old mode 100755 new mode 100644 diff --git a/src/associate_levelpaths_with_levees.py b/src/associate_levelpaths_with_levees.py old mode 100755 new mode 100644 diff --git a/src/bash_functions.env b/src/bash_functions.env old mode 100755 new mode 100644 diff --git a/src/bash_variables.env b/src/bash_variables.env old mode 100755 new mode 100644 diff --git a/src/bathy_src_adjust_topwidth.py b/src/bathy_src_adjust_topwidth.py old mode 100755 new mode 100644 diff --git a/src/bathymetric_adjustment.py b/src/bathymetric_adjustment.py old mode 100755 new mode 100644 diff --git a/src/build_stream_traversal.py b/src/build_stream_traversal.py old mode 100755 new mode 100644 diff --git a/src/check_unit_errors.py b/src/check_unit_errors.py old mode 100755 new mode 100644 diff --git a/src/derive_headwaters.py b/src/derive_headwaters.py old mode 100755 new mode 100644 diff --git a/src/entrypoint.sh b/src/entrypoint.sh old mode 100755 new mode 100644 diff --git a/src/src_adjust_ras2fim_rating.py b/src/src_adjust_ras2fim_rating.py old mode 100755 new mode 100644 diff --git a/src/src_adjust_spatial_obs.py b/src/src_adjust_spatial_obs.py old mode 100755 new mode 100644 diff --git a/src/src_adjust_usgs_rating.py b/src/src_adjust_usgs_rating.py old mode 100755 new mode 100644 diff --git a/src/src_manual_calibration.py b/src/src_manual_calibration.py old mode 100755 new mode 100644 diff --git a/src/src_roughness_optimization.py b/src/src_roughness_optimization.py old mode 100755 new mode 100644 diff --git a/src/utils/__init__.py b/src/utils/__init__.py old mode 100755 new mode 100644 diff --git a/src/utils/fim_enums.py b/src/utils/fim_enums.py old mode 100755 new mode 100644 diff --git a/src/utils/shared_functions.py b/src/utils/shared_functions.py old mode 100755 new mode 100644 diff --git a/src/utils/shared_validators.py b/src/utils/shared_validators.py old mode 100755 new mode 100644 diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py old mode 100755 new mode 100644 diff --git a/tools/.env.template b/tools/.env.template old mode 100755 new mode 100644 diff --git a/tools/.gitignore b/tools/.gitignore old mode 100755 new mode 100644 diff --git a/tools/adjust_rc_with_feedback.py b/tools/adjust_rc_with_feedback.py old mode 100755 new mode 100644 diff --git a/tools/aggregate_csv_files.py b/tools/aggregate_csv_files.py old mode 100755 new mode 100644 diff --git a/tools/check_deep_flooding.py b/tools/check_deep_flooding.py old mode 100755 new mode 100644 diff --git a/tools/combine_crosswalk_tables.py b/tools/combine_crosswalk_tables.py old mode 100755 new mode 100644 diff --git a/tools/compare_ms_and_non_ms_metrics.py b/tools/compare_ms_and_non_ms_metrics.py old mode 100755 new mode 100644 diff --git a/tools/composite_inundation.py b/tools/composite_inundation.py old mode 100755 new mode 100644 diff --git a/tools/eval_alt_catfim.py b/tools/eval_alt_catfim.py old mode 100755 new mode 100644 diff --git a/tools/eval_plots.py b/tools/eval_plots.py old mode 100755 new mode 100644 diff --git a/tools/eval_plots_stackedbar.py b/tools/eval_plots_stackedbar.py old mode 100755 new mode 100644 diff --git a/tools/evaluate_continuity.py b/tools/evaluate_continuity.py old mode 100755 new mode 100644 diff --git a/tools/fimr_to_benchmark.py b/tools/fimr_to_benchmark.py old mode 100755 new mode 100644 diff --git a/tools/find_test_case_folders.py b/tools/find_test_case_folders.py old mode 100755 new mode 100644 diff --git a/tools/generate_nws_lid.py b/tools/generate_nws_lid.py old mode 100755 new mode 100644 diff --git a/tools/inundate_mosaic_wrapper.py b/tools/inundate_mosaic_wrapper.py old mode 100755 new mode 100644 diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index 050436670..d929cf2ad 100755 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -6,6 +6,7 @@ import os import re import shutil +import sys from datetime import datetime from multiprocessing import Pool @@ -14,6 +15,7 @@ from osgeo import gdal from utils.shared_functions import FIM_Helpers as fh +from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv # INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/' @@ -113,7 +115,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(msg) # Perform VRT creation and mosaic all of the huc rasters using boolean rasters - vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name, job_number) + vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name,job_number) # now cleanup the temp bool directory shutil.rmtree(output_bool_dir, ignore_errors=True) @@ -228,7 +230,7 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads): vrt, xRes=10, yRes=-10, - creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS=' + str(threads)], + creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2','NUM_THREADS='+str(threads)], ) vrt = None diff --git a/tools/overlapping_inundation.py b/tools/overlapping_inundation.py old mode 100755 new mode 100644 diff --git a/tools/pixel_counter.py b/tools/pixel_counter.py old mode 100755 new mode 100644 diff --git a/tools/pixel_counter_functions.py b/tools/pixel_counter_functions.py old mode 100755 new mode 100644 diff --git a/tools/pixel_counter_wrapper.py b/tools/pixel_counter_wrapper.py old mode 100755 new mode 100644 diff --git a/tools/rating_curve_get_usgs_curves.py b/tools/rating_curve_get_usgs_curves.py old mode 100755 new mode 100644 diff --git a/tools/test_case_by_hydro_id.py b/tools/test_case_by_hydro_id.py old mode 100755 new mode 100644 diff --git a/unit_tests/README.md b/unit_tests/README.md old mode 100755 new mode 100644 diff --git a/unit_tests/__template.json b/unit_tests/__template.json old mode 100755 new mode 100644 diff --git a/unit_tests/__template.py b/unit_tests/__template.py old mode 100755 new mode 100644 diff --git a/unit_tests/aggregate_branch_lists_params.json b/unit_tests/aggregate_branch_lists_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/aggregate_branch_lists_test.py b/unit_tests/aggregate_branch_lists_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/check_unit_errors_params.json b/unit_tests/check_unit_errors_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/check_unit_errors_test.py b/unit_tests/check_unit_errors_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/clip_vectors_to_wbd_params.json b/unit_tests/clip_vectors_to_wbd_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/clip_vectors_to_wbd_test.py b/unit_tests/clip_vectors_to_wbd_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/derive_level_paths_params.json b/unit_tests/derive_level_paths_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/derive_level_paths_test.py b/unit_tests/derive_level_paths_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/filter_catchments_and_add_attributes_params.json b/unit_tests/filter_catchments_and_add_attributes_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/filter_catchments_and_add_attributes_test.py b/unit_tests/filter_catchments_and_add_attributes_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/generate_branch_list_csv_params.json b/unit_tests/generate_branch_list_csv_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/generate_branch_list_csv_test.py b/unit_tests/generate_branch_list_csv_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/generate_branch_list_params.json b/unit_tests/generate_branch_list_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/generate_branch_list_test.py b/unit_tests/generate_branch_list_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/inundate_gms_params.json b/unit_tests/inundate_gms_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/inundate_gms_test.py b/unit_tests/inundate_gms_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/inundation_params.json b/unit_tests/inundation_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/inundation_test.py b/unit_tests/inundation_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/outputs_cleanup_params.json b/unit_tests/outputs_cleanup_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/outputs_cleanup_test.py b/unit_tests/outputs_cleanup_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/pyproject.toml b/unit_tests/pyproject.toml old mode 100755 new mode 100644 diff --git a/unit_tests/rating_curve_comparison_params.json b/unit_tests/rating_curve_comparison_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/rating_curve_comparison_test.py b/unit_tests/rating_curve_comparison_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/shared_functions_params.json b/unit_tests/shared_functions_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/shared_functions_test.py b/unit_tests/shared_functions_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/split_flows_params.json b/unit_tests/split_flows_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/split_flows_test.py b/unit_tests/split_flows_test.py old mode 100755 new mode 100644 diff --git a/unit_tests/unit_tests_utils.py b/unit_tests/unit_tests_utils.py old mode 100755 new mode 100644 diff --git a/unit_tests/usgs_gage_crosswalk_params.json b/unit_tests/usgs_gage_crosswalk_params.json old mode 100755 new mode 100644 diff --git a/unit_tests/usgs_gage_crosswalk_test.py b/unit_tests/usgs_gage_crosswalk_test.py old mode 100755 new mode 100644 From 205525ee33551e06bd6515664bff954b61184a6b Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Fri, 1 Dec 2023 17:48:16 +0000 Subject: [PATCH 18/19] fixing linting errors --- tools/inundate_nation.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index d929cf2ad..6eda62eda 100755 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -6,7 +6,6 @@ import os import re import shutil -import sys from datetime import datetime from multiprocessing import Pool @@ -15,7 +14,6 @@ from osgeo import gdal from utils.shared_functions import FIM_Helpers as fh -from utils.shared_variables import PREP_PROJECTION, elev_raster_ndv # INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/' @@ -115,7 +113,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, logging.info(msg) # Perform VRT creation and mosaic all of the huc rasters using boolean rasters - vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name,job_number) + vrt_raster_mosaic(output_bool_dir, output_dir, output_base_file_name, job_number) # now cleanup the temp bool directory shutil.rmtree(output_bool_dir, ignore_errors=True) @@ -230,7 +228,7 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads): vrt, xRes=10, yRes=-10, - creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2','NUM_THREADS='+str(threads)], + creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS='+str(threads)], ) vrt = None From f71e0af4012758ef02e6b3f0d864628a53a9dc3b Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Fri, 1 Dec 2023 18:57:41 +0000 Subject: [PATCH 19/19] more linting fixes --- tools/inundate_nation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/inundate_nation.py b/tools/inundate_nation.py index 6eda62eda..050436670 100755 --- a/tools/inundate_nation.py +++ b/tools/inundate_nation.py @@ -228,7 +228,7 @@ def vrt_raster_mosaic(output_bool_dir, output_dir, fim_version_tag, threads): vrt, xRes=10, yRes=-10, - creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS='+str(threads)], + creationOptions=['COMPRESS=LZW', 'TILED=YES', 'PREDICTOR=2', 'NUM_THREADS=' + str(threads)], ) vrt = None