From b3d2ac40000f29c565430c41e2311553de66ed21 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 5 Feb 2024 09:43:02 -0500 Subject: [PATCH 01/11] remove gdal version restriction --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b2583b0f..fe0d6f5d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2,<3.6.0 +GDAL>=3.4.2 Pyro4==4.77 # pip-only pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 From a5a3d17446bab18bc42ea0c14f97cb3254f59752 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 5 Feb 2024 10:57:35 -0500 Subject: [PATCH 02/11] awy output spec had intermediate dir at wrong level --- src/natcap/invest/annual_water_yield.py | 106 ++++++++++++------------ 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/natcap/invest/annual_water_yield.py b/src/natcap/invest/annual_water_yield.py index b0ebf1e9d..8e4ccd136 100644 --- a/src/natcap/invest/annual_water_yield.py +++ b/src/natcap/invest/annual_water_yield.py @@ -376,61 +376,61 @@ } } } + } + } + }, + "intermediate": { + "type": "directory", + "contents": { + "clipped_lulc.tif": { + "about": "Aligned and clipped copy of LULC input.", + "bands": {1: {"type": "integer"}} }, - "intermediate": { - "type": "directory", - "contents": { - "clipped_lulc.tif": { - "about": "Aligned and clipped copy of LULC input.", - "bands": {1: {"type": "integer"}} - }, - "depth_to_root_rest_layer.tif": { - "about": ( - "Aligned and clipped copy of root restricting " - "layer depth input."), - "bands": { - 1: {"type": "number", "units": u.millimeter} - } - }, - "eto.tif": { - "about": "Aligned and clipped copy of ET0 input.", - "bands": { - 1: {"type": "number", "units": u.millimeter} - } - }, - "kc_raster.tif": { - "about": "Map of KC values.", - "bands": { - 1: {"type": "number", "units": u.none} - } - }, - "pawc.tif": { - "about": "Aligned and clipped copy of PAWC input.", - "bands": {1: {"type": "ratio"}}, - }, - "pet.tif": { - "about": "Map of potential evapotranspiration.", - "bands": { - 1: {"type": "number", "units": u.millimeter} - } - }, - "precip.tif": { - "about": "Aligned and clipped copy of precipitation input.", - "bands": { - 1: {"type": "number", "units": u.millimeter} - } - }, - "root_depth.tif": { - "about": "Map of root depth.", - "bands": { - 1: {"type": "number", "units": u.millimeter} - } - }, - "veg.tif": { - "about": "Map of vegetated state.", - "bands": {1: {"type": "integer"}}, - } + "depth_to_root_rest_layer.tif": { + "about": ( + "Aligned and clipped copy of root restricting " + "layer depth input."), + "bands": { + 1: {"type": "number", "units": u.millimeter} } + }, + "eto.tif": { + "about": "Aligned and clipped copy of ET0 input.", + "bands": { + 1: {"type": "number", "units": u.millimeter} + } + }, + "kc_raster.tif": { + "about": "Map of KC values.", + "bands": { + 1: {"type": "number", "units": u.none} + } + }, + "pawc.tif": { + "about": "Aligned and clipped copy of PAWC input.", + "bands": {1: {"type": "ratio"}}, + }, + "pet.tif": { + "about": "Map of potential evapotranspiration.", + "bands": { + 1: {"type": "number", "units": u.millimeter} + } + }, + "precip.tif": { + "about": "Aligned and clipped copy of precipitation input.", + "bands": { + 1: {"type": "number", "units": u.millimeter} + } + }, + "root_depth.tif": { + "about": "Map of root depth.", + "bands": { + 1: {"type": "number", "units": u.millimeter} + } + }, + "veg.tif": { + "about": "Map of vegetated state.", + "bands": {1: {"type": "integer"}}, } } }, From 8f5f0538597e164ff46cfd45556bb54db8933e2a Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 12 Feb 2024 09:44:22 -0500 Subject: [PATCH 03/11] fix some filenames in output spec. --- src/natcap/invest/carbon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/natcap/invest/carbon.py b/src/natcap/invest/carbon.py index 9752426c8..7c3683180 100644 --- a/src/natcap/invest/carbon.py +++ b/src/natcap/invest/carbon.py @@ -21,7 +21,7 @@ LOGGER = logging.getLogger(__name__) CARBON_OUTPUTS = { - f"c_{scenario}_{pool}.tif": { + f"c_{pool}_{scenario}.tif": { "about": ( f"Raster of {pool_name} carbon values in the {scenario_name} " "scenario, mapped from the Carbon Pools table to the LULC."), @@ -252,7 +252,7 @@ }}, "created_if": "lulc_redd_path" }, - "intermediate": { + "intermediate_outputs": { "type": "directory", "contents": { **CARBON_OUTPUTS From 66f657554235f8367664a00350d968c9f1b747e8 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 12 Feb 2024 09:48:52 -0500 Subject: [PATCH 04/11] restrict scipy version. #1508 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b2583b0f..eb80fec07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 -scipy>=1.9.0 +scipy>=1.9.0,<1.12.0 pygeoprocessing>=2.4.2 # pip-only taskgraph>=0.11.0 psutil>=5.6.6 From 04df0bd4f4d60d88d8a4959ead1db3361489e698 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 09:13:28 -0500 Subject: [PATCH 05/11] scipy patch was applied and should be in included in 1.13.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb80fec07..cad4fa0e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 -scipy>=1.9.0,<1.12.0 +scipy>=1.9.0,!=1.12.* pygeoprocessing>=2.4.2 # pip-only taskgraph>=0.11.0 psutil>=5.6.6 From 973aea8919c57291bcc6b7c7f2747083b31027ce Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 09:19:15 -0500 Subject: [PATCH 06/11] Revert "remove gdal version restriction" This reverts commit b3d2ac40000f29c565430c41e2311553de66ed21. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe0d6f5d0..5b2583b0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2 +GDAL>=3.4.2,<3.6.0 Pyro4==4.77 # pip-only pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 From 6530c6db1b1b2e960010191bdff40aaf10fc958d Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 09:33:00 -0500 Subject: [PATCH 07/11] no longer need to restrict GDAL. #1508 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cad4fa0e7..d20cae2cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2,<3.6.0 +GDAL>=3.4.2 Pyro4==4.77 # pip-only pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 From 97a16945a9734af1e0284c9198fc496c64787233 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 09:47:01 -0500 Subject: [PATCH 08/11] including the results_suffix in a few intermediate files that are no longer temp files. #1517 --- src/natcap/invest/annual_water_yield.py | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/natcap/invest/annual_water_yield.py b/src/natcap/invest/annual_water_yield.py index 8e4ccd136..fdaeaa862 100644 --- a/src/natcap/invest/annual_water_yield.py +++ b/src/natcap/invest/annual_water_yield.py @@ -576,8 +576,12 @@ def execute(args): wyield_path = os.path.join( per_pixel_output_dir, f'wyield{file_suffix}.tif') aet_path = os.path.join(per_pixel_output_dir, f'aet{file_suffix}.tif') - demand_path = os.path.join(intermediate_dir, f'demand{file_suffix}.tif') + veg_raster_path = os.path.join(intermediate_dir, f'veg{file_suffix}.tif') + root_raster_path = os.path.join( + intermediate_dir, f'root_depth{file_suffix}.tif') + kc_raster_path = os.path.join( + intermediate_dir, f'kc_raster{file_suffix}.tif') watersheds_path = args['watersheds_path'] watershed_results_vector_path = os.path.join( @@ -694,39 +698,35 @@ def execute(args): 'table_name': 'Biophysical'} # Create Kc raster from table values to use in future calculations LOGGER.info("Reclassifying temp_Kc raster") - tmp_Kc_raster_path = os.path.join(intermediate_dir, 'kc_raster.tif') create_Kc_raster_task = graph.add_task( func=utils.reclassify_raster, - args=((clipped_lulc_path, 1), Kc_dict, tmp_Kc_raster_path, + args=((clipped_lulc_path, 1), Kc_dict, kc_raster_path, gdal.GDT_Float32, nodata_dict['out_nodata'], reclass_error_details), - target_path_list=[tmp_Kc_raster_path], + target_path_list=[kc_raster_path], dependent_task_list=[align_raster_stack_task], task_name='create_Kc_raster') # Create root raster from table values to use in future calculations LOGGER.info("Reclassifying tmp_root raster") - tmp_root_raster_path = os.path.join( - intermediate_dir, 'root_depth.tif') create_root_raster_task = graph.add_task( func=utils.reclassify_raster, - args=((clipped_lulc_path, 1), root_dict, tmp_root_raster_path, + args=((clipped_lulc_path, 1), root_dict, root_raster_path, gdal.GDT_Float32, nodata_dict['out_nodata'], reclass_error_details), - target_path_list=[tmp_root_raster_path], + target_path_list=[root_raster_path], dependent_task_list=[align_raster_stack_task], task_name='create_root_raster') # Create veg raster from table values to use in future calculations # of determining which AET equation to use LOGGER.info("Reclassifying tmp_veg raster") - tmp_veg_raster_path = os.path.join(intermediate_dir, 'veg.tif') create_veg_raster_task = graph.add_task( func=utils.reclassify_raster, - args=((clipped_lulc_path, 1), vegetated_dict, tmp_veg_raster_path, + args=((clipped_lulc_path, 1), vegetated_dict, veg_raster_path, gdal.GDT_Float32, nodata_dict['out_nodata'], reclass_error_details), - target_path_list=[tmp_veg_raster_path], + target_path_list=[veg_raster_path], dependent_task_list=[align_raster_stack_task], task_name='create_veg_raster') @@ -737,7 +737,7 @@ def execute(args): func=pygeoprocessing.raster_map, kwargs=dict( op=numpy.multiply, # PET = ET0 * KC - rasters=[eto_path, tmp_Kc_raster_path], + rasters=[eto_path, kc_raster_path], target_path=tmp_pet_path, target_nodata=nodata_dict['out_nodata']), target_path_list=[tmp_pet_path], @@ -747,8 +747,8 @@ def execute(args): # List of rasters to pass into the vectorized fractp operation raster_list = [ - tmp_Kc_raster_path, eto_path, precip_path, tmp_root_raster_path, - depth_to_root_rest_layer_path, pawc_path, tmp_veg_raster_path] + kc_raster_path, eto_path, precip_path, root_raster_path, + depth_to_root_rest_layer_path, pawc_path, veg_raster_path] LOGGER.debug('Performing fractp operation') calculate_fractp_task = graph.add_task( From 9086b64a0dd284c1cb594c8eff695f398240567f Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 09:56:12 -0500 Subject: [PATCH 09/11] note for HISTORY. #1517 --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 0853f86a3..cf88c717b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -38,6 +38,9 @@ Unreleased Changes ------------------ +* Annual Water Yield + * Added the results_suffix to a few intermediate files where it was + missing. https://github.com/natcap/invest/issues/1517 * Urban Nature Access * Fixed a ``NameError`` that occurred when running the model using search radii defined per population group with an exponential search From a2c916105cdb7232210cacb19c08dc2799d765d6 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 10:03:24 -0500 Subject: [PATCH 10/11] restrict gdal to until dealing with DeprecationWarnings. #1508 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d20cae2cf..2daf7319f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2 +GDAL>=3.4.2,<3.8.0 Pyro4==4.77 # pip-only pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 From 9df565c6e2b43198bd2321137f2e677979ee5571 Mon Sep 17 00:00:00 2001 From: davemfish Date: Mon, 19 Feb 2024 10:16:05 -0500 Subject: [PATCH 11/11] revert the gdal version change. #1508 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2daf7319f..cad4fa0e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2,<3.8.0 +GDAL>=3.4.2,<3.6.0 Pyro4==4.77 # pip-only pandas>=1.2.1 numpy>=1.11.0,!=1.16.0