Merge remote-tracking branch 'origin/dev' into dev-catfim-alaska

Dockerfile

@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -45,7 +45,7 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 
 ###############################################################################################
 # Base Image that has GDAL, PROJ, etc
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
@@ -107,7 +107,7 @@ ENV PYTHONPATH=${PATH}:$srcDir:$projectDir/tools
 
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
 
 # ----------------------------------
 # Mar 2023

Pipfile

@@ -7,53 +7,53 @@ verify_ssl = true
 ipython = "==8.24.0"
 
 [packages]
+black = "==24.3.0"
+boto3 = "1.35.1"
 certifi = "==2024.7.4"
-fiona = "==1.8.22"
-geopandas = "==0.14.3"
+dask = "==2024.6.2"
+dask-expr = "==1.1.6"
+distributed = "==2024.6.2"
+fiona = "==1.9.6"
+flake8 = "==6.0.0"
+flake8-pyproject = "==1.2.3"
+geopandas = "==1.0.1"
+gval = "==0.2.7.post1"
+ipympl = "==0.9.3"
+isort = "==5.12.0"
+jupyter = "==1.0.0"
+jupyterlab = "==3.6.7"
+lmoments3 = "==1.0.6"
+monaco = "==0.13.1"
+natsort = "==8.3.1"
+netcdf4 = "==1.6.3"
 numba = "==0.60.0"
 numpy = "==1.26.4"
+openpyxl = "==3.1.2"
+osmnx = "==1.9.3"
 pandas = "==2.0.2"
-rasterio = "==1.3.6"
-rasterstats = "==0.18.0"
-richdem = "==0.3.4"
-tqdm = "==4.66.3"
-seaborn = "==0.12.2"
-python-dotenv = "==1.0.0"
-natsort = "==8.3.1"
-xarray = "==2024.6.0"
-netcdf4 = "==1.6.3"
-tables = "==3.8.0"
-pyproj = "==3.5.0"
+pillow = "==10.3.0"
+pre-commit = "==3.3.3"
 psycopg2-binary = "==2.9.6"
-boto3 = "==1.26.109"
-jupyter = "==1.0.0"
-jupyterlab = "==3.6.7"
-ipympl = "==0.9.3"
-whitebox = "==2.3.4"
-shapely = "==2.0.1"
-pyarrow = "==14.0.1"
-rtree = "==1.0.1"
+pyarrow = "==17.0.0"
+pyflwdir = "==0.5.8"
+pyogrio = "==0.8.0"
+pyproj = "==3.6.1"
+python-dotenv = "==1.0.0"
 py7zr = "==0.20.4"
+rasterio = "==1.3.10"
+rasterstats = "==0.19.0"
+requests = "==2.32.3"
+richdem = "==0.3.4"
+rtree = "==1.0.1"
 scipy = "==1.14.0"
-gval = "==0.2.7"
-flake8 = "==6.0.0"
-black = "==24.3.0"
-flake8-pyproject = "==1.2.3"
-pre-commit = "==3.3.3"
-isort = "==5.12.0"
+seaborn = "==0.12.2"
+tables = "==3.8.0"
+tqdm = "==4.66.3"
 urllib3 = "==1.26.19"
-pyflwdir = "==0.5.8"
-pillow = "==10.3.0"
-pyogrio = "==0.7.2"
-openpyxl = "==3.1.2"
-osmnx = "==1.9.3"
-lmoments3 = "==1.0.6"
+whitebox = "==2.3.4"
+xarray = "==2024.6.0"
 zarr = "==2.18.0"
-requests = "==2.32.3"
-aiohttp = "==3.9.5"
-distributed = "==2024.6.2"
-monaco = "==0.13.1"
-dask = "==2024.6.2"
+
 
 [requires]
 python_version = "3.10"

docs/CHANGELOG.md

@@ -1,6 +1,44 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.6.0 - 2024-08-23 - [PR#1253](https://github.com/NOAA-OWP/inundation-mapping/pull/1253)
+
+Upgrades Python packages and dependencies and fixes backwards incompatibilities with new version of `geopandas`. Major changes include:
+- Upgrading `boto3`, `fiona`, `geopandas`, `gval`, `pyarrow`, `pyogrio`, `pyproj`, and `rasterio`
+- Removing `aiobotocore` and `aiohttp`
+
+### Changes
+
+- `Dockerfile`: Upgrade GDAL (v3.8.3) and pipenv
+- `Pipfile` and `Pipfile.lock`: Upgrade Python packages, add dask-expr, and remove aiohttp
+- `src/`
+    - `build_stream_traversal.py`, `derive_level_paths.py`, `heal_bridges_osm.py`, `mitigate_branch_outlet_backpool.py`, `split_flows.py`, `stream_branches.py`, `usgs_gage_unit_setup.py`: Fix backwards incompatibilities with new version of `geopandas`.
+
+<br/><br/>
+
+## v4.5.5.1 - 2024-08-16 - [PR#1225](https://github.com/NOAA-OWP/inundation-mapping/pull/1225)
+
+Removes warning when running `heal_bridges_osm.py` by not saving the empty DataFrame.
+
+### Changes
+
+- `src/heal_bridges_osm.py`
+
+<br/><br/>
+
+
+## v4.5.5.0 - 2024-08-16 - [PR#1247](https://github.com/NOAA-OWP/inundation-mapping/pull/1247)
+
+Updated the gauge crosswalk and SRC adjustment routine to use the ras2fim v2 files. The v2 ras2fim file structure was changed to organize the data by huc8 - one gpkg and csv per huc8. Addresses #1091 
+
+### Changes
+- `fim_post_processing.sh`: added new input variables for running the `src_adjust_ras2fim_rating.py`
+- `src/bash_variables.env`: renamed and reassigned the ras2fim input variables: `ras2fim_input_dir`, `ras_rating_curve_csv_filename`, `ras_rating_curve_gpkg_filename`
+- `src/run_unit_wb.sh`: Added logic to check if huc in process has ras2fim input data to process. If yes - copy the ras2fim cross section point gpkg to the huc run directory.
+- `src/src_adjust_ras2fim_rating.py`: Updated code logic to use the huc-specific input files containing the ras2fim rating curve data (previous ras2fim input file contained all hucs in one csv)
+- `src/utils/shared_functions.py`: Added function to find huc subdirectories with the same name btw two parent folders
+
+ <br/><br/>
 
 ## v4.5.4.4 - 2024-08-02 - [PR#1238](https://github.com/NOAA-OWP/inundation-mapping/pull/1238)
 

fim_post_processing.sh

@@ -217,7 +217,8 @@ if [ "$src_adjust_ras2fim" = "True" ] && [ "$src_subdiv_toggle" = "True" ] && [
     # Run SRC Optimization routine using ras2fim rating curve data (WSE and flow @ NWM recur flow values)
     python3 $srcDir/src_adjust_ras2fim_rating.py \
         -run_dir $outputDestDir \
-        -ras_rc $ras_rating_curve_csv \
+        -ras_input $ras2fim_input_dir \
+        -ras_rc $ras_rating_curve_csv_filename \
         -nwm_recur $nwm_recur_file \
         -j $jobLimit
     Tcount

src/bash_variables.env

@@ -47,9 +47,9 @@ export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flo
 export             usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
 
 # input file locations for ras2fim locations and rating curve data
-export              ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
-export      ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
-export    ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points_rel_101.gpkg
+export                  ras2fim_input_dir=${inputsDir}/rating_curve/ras2fim_exports/v2_0
+export      ras_rating_curve_csv_filename=reformat_ras_rating_curve_table.csv
+export     ras_rating_curve_gpkg_filename=reformat_ras_rating_curve_points.gpkg
 
 export                  fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
 

src/build_stream_traversal.py

@@ -64,9 +64,7 @@ def execute(self, streams, wbd8, hydro_id):
                     {'geometry': stream_midpoint}, crs=streams.crs, geometry='geometry'
                 )
                 stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', predicate='within')
-                stream_wbdjoin = stream_wbdjoin.rename(
-                    columns={"geometry": "midpoint", "index_right": "HUC8id"}
-                )
+                stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "fimid": "HUC8id"})
                 streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
                 streams['seqID'] = (

src/derive_level_paths.py

@@ -192,7 +192,10 @@ def Derive_level_paths(
         )
 
         stream_network = stream_network.remove_branches_in_waterbodies(
-            waterbodies=waterbodies, out_vector_files=out_stream_network_dissolved, verbose=False
+            waterbodies=waterbodies,
+            out_vector_files=out_stream_network_dissolved,
+            branch_id_attribute=branch_id_attribute,
+            verbose=False,
         )
         stream_network = stream_network.select_branches_intersecting_huc(
             wbd=wbd,

src/heal_bridges_osm.py

@@ -37,7 +37,7 @@ def process_bridges_in_huc(
 
         # Get max hand values for each bridge
         osm_gdf['max_hand'] = zonal_stats(
-            osm_gdf['geometry'], hand_grid_array, affine=hand_grid.transform, stats="max"
+            osm_gdf['geometry'], hand_grid_array, affine=hand_grid.transform, stats="max", nodata=-999
         )
         # pull the values out of the geopandas columns so we can use them as floats
         osm_gdf['max_hand'] = [x.get('max') for x in osm_gdf.max_hand]
@@ -71,9 +71,12 @@ def process_bridges_in_huc(
     osm_gdf['branch'] = branch_id
     osm_gdf['mainstem'] = 0 if branch_id == '0' else 1
 
-    # Write the bridge points to a geopackage
-    osm_gdf.to_file(bridge_centroids, index=False)
-
+    # Check if the GeoDataFrame is empty
+    if not osm_gdf.empty:
+        # Write the bridge points to a geopackage
+        osm_gdf.to_file(bridge_centroids, index=False, engine='fiona')
+    else:
+        print('The geoDataFrame is empty. File not saved.')
     return
 
 
@@ -100,6 +103,7 @@ def flows_from_hydrotable(bridge_pnts, hydroTable):
         python3 src/heal_bridges_osm.py
             -g /outputs/fim_4_4_15_0/1209301/branches/3763000013/rem_zeroed_masked_3763000013.tif
             -s /outputs/fim_4_4_15_0/1209301/osm_bridges_subset.gpkg
+            -p /outputs/fim_4_4_15_0/1209301/branches/3763000013/gw_catchments_reaches_filtered_addedAttributes_crosswalked_3763000013.gpkg
             -c /outputs/fim_4_4_15_0/1209301/1209301/branches/3763000013/osm_bridge_centroids_3763000013.tif
             -b 10
             -r 10

src/mitigate_branch_outlet_backpool.py

@@ -390,7 +390,7 @@ def mask_raster_to_boundary(raster_path, boundary_json, save_path):
                     trimmed_flows, inital_length_km = snap_and_trim_splitflow(pt_3tl_geom, split_flows_geom)
 
                     # Create buffer around the updated flows geodataframe (and make sure it's all one shape)
-                    buffer = trimmed_flows.buffer(10).geometry.unary_union
+                    buffer = trimmed_flows.buffer(10).geometry.union_all()
 
                     # Remove flowpoints that don't intersect with the trimmed flow line
                     split_points_filtered_geom = split_points_geom[split_points_geom.geometry.within(buffer)]

src/run_unit_wb.sh

@@ -48,9 +48,25 @@ cp -a $pre_clip_huc_dir/$hucNumber/. $tempHucDataDir
 # For buffer_stream_branches.py
 cp $huc_input_DEM_domain $tempHucDataDir
 # For usgs_gage_unit_setup.py
-cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
-cp $ras_rating_curve_points_gpkg $tempHucDataDir
 cp $inputsDir/ahps_sites/nws_lid.gpkg $tempHucDataDir
+cp $inputsDir/usgs_gages/usgs_gages.gpkg $tempHucDataDir
+# Check if the $hucNumber directory exists in the ras2fim $inputsDir
+if [ -d "$ras2fim_input_dir/$hucNumber" ]; then
+    ras_rating_gpkg="$ras2fim_input_dir/$hucNumber/$ras_rating_curve_gpkg_filename"
+    ras_rating_csv="$ras2fim_input_dir/$hucNumber/$ras_rating_curve_csv_filename"
+    if [ -f "$ras_rating_gpkg" ]; then
+        cp "$ras_rating_gpkg" $tempHucDataDir
+        echo "Copied $ras_rating_gpkg to $tempHucDataDir"
+    else
+        echo "File $ras_rating_gpkg does not exist. Skipping copy."
+    fi
+    if [ -f "$ras_rating_csv" ]; then
+        cp "$ras_rating_csv" $tempHucDataDir
+        echo "Copied $ras_rating_csv to $tempHucDataDir"
+    else
+        echo "File $ras_rating_csv does not exist. Skipping copy."
+    fi
+fi
 
 ## DERIVE LEVELPATH  ##
 echo -e $startDiv"Generating Level Paths for $hucNumber"

src/split_flows.py

@@ -370,7 +370,7 @@ def snap_and_trim_flow(snapped_point, flows):
         split_flows_gdf = gpd.sjoin(
             split_flows_gdf, lakes_buffer, how='left', predicate='within'
         )  # Note: Options include intersects, within, contains, crosses
-        split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
+        split_flows_gdf = split_flows_gdf.rename(columns={"newID": "LakeID"}).fillna(-999)
     else:
         split_flows_gdf['LakeID'] = -999