v4.5.6.0 Upgrade Python packages (#1253)

Dockerfile

@@ -1,5 +1,5 @@
 ## Temporary image to build the libraries and only save the needed artifacts
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3 AS builder
 WORKDIR /opt/builder
 ARG dataDir=/data
 ARG projectDir=/foss_fim
@@ -45,7 +45,7 @@ RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d
 
 ###############################################################################################
 # Base Image that has GDAL, PROJ, etc
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.0
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.8.3
 ARG dataDir=/data
 ENV projectDir=/foss_fim
 ARG depDir=/dependencies
@@ -107,7 +107,7 @@ ENV PYTHONPATH=${PATH}:$srcDir:$projectDir/tools
 
 COPY Pipfile .
 COPY Pipfile.lock .
-RUN pip3 install pipenv==2023.12.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
+RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --system --deploy --ignore-pipfile
 
 # ----------------------------------
 # Mar 2023

Pipfile

@@ -7,53 +7,53 @@ verify_ssl = true
 ipython = "==8.24.0"
 
 [packages]
+black = "==24.3.0"
+boto3 = "1.35.1"
 certifi = "==2024.7.4"
-fiona = "==1.8.22"
-geopandas = "==0.14.3"
+dask = "==2024.6.2"
+dask-expr = "==1.1.6"
+distributed = "==2024.6.2"
+fiona = "==1.9.6"
+flake8 = "==6.0.0"
+flake8-pyproject = "==1.2.3"
+geopandas = "==1.0.1"
+gval = "==0.2.7.post1"
+ipympl = "==0.9.3"
+isort = "==5.12.0"
+jupyter = "==1.0.0"
+jupyterlab = "==3.6.7"
+lmoments3 = "==1.0.6"
+monaco = "==0.13.1"
+natsort = "==8.3.1"
+netcdf4 = "==1.6.3"
 numba = "==0.60.0"
 numpy = "==1.26.4"
+openpyxl = "==3.1.2"
+osmnx = "==1.9.3"
 pandas = "==2.0.2"
-rasterio = "==1.3.6"
-rasterstats = "==0.18.0"
-richdem = "==0.3.4"
-tqdm = "==4.66.3"
-seaborn = "==0.12.2"
-python-dotenv = "==1.0.0"
-natsort = "==8.3.1"
-xarray = "==2024.6.0"
-netcdf4 = "==1.6.3"
-tables = "==3.8.0"
-pyproj = "==3.5.0"
+pillow = "==10.3.0"
+pre-commit = "==3.3.3"
 psycopg2-binary = "==2.9.6"
-boto3 = "==1.26.109"
-jupyter = "==1.0.0"
-jupyterlab = "==3.6.7"
-ipympl = "==0.9.3"
-whitebox = "==2.3.4"
-shapely = "==2.0.1"
-pyarrow = "==14.0.1"
-rtree = "==1.0.1"
+pyarrow = "==17.0.0"
+pyflwdir = "==0.5.8"
+pyogrio = "==0.8.0"
+pyproj = "==3.6.1"
+python-dotenv = "==1.0.0"
 py7zr = "==0.20.4"
+rasterio = "==1.3.10"
+rasterstats = "==0.19.0"
+requests = "==2.32.3"
+richdem = "==0.3.4"
+rtree = "==1.0.1"
 scipy = "==1.14.0"
-gval = "==0.2.7"
-flake8 = "==6.0.0"
-black = "==24.3.0"
-flake8-pyproject = "==1.2.3"
-pre-commit = "==3.3.3"
-isort = "==5.12.0"
+seaborn = "==0.12.2"
+tables = "==3.8.0"
+tqdm = "==4.66.3"
 urllib3 = "==1.26.19"
-pyflwdir = "==0.5.8"
-pillow = "==10.3.0"
-pyogrio = "==0.7.2"
-openpyxl = "==3.1.2"
-osmnx = "==1.9.3"
-lmoments3 = "==1.0.6"
+whitebox = "==2.3.4"
+xarray = "==2024.6.0"
 zarr = "==2.18.0"
-requests = "==2.32.3"
-aiohttp = "==3.9.5"
-distributed = "==2024.6.2"
-monaco = "==0.13.1"
-dask = "==2024.6.2"
+
 
 [requires]
 python_version = "3.10"

docs/CHANGELOG.md

@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.6.0 - 2024-08-23 - [PR#1253](https://github.com/NOAA-OWP/inundation-mapping/pull/1253)
+
+Upgrades Python packages and dependencies and fixes backwards incompatibilities with new version of `geopandas`. Major changes include:
+- Upgrading `boto3`, `fiona`, `geopandas`, `gval`, `pyarrow`, `pyogrio`, `pyproj`, and `rasterio`
+- Removing `aiobotocore` and `aiohttp`
+
+### Changes
+
+- `Dockerfile`: Upgrade GDAL (v3.8.3) and pipenv
+- `Pipfile` and `Pipfile.lock`: Upgrade Python packages, add dask-expr, and remove aiohttp
+- `src/`
+    - `build_stream_traversal.py`, `derive_level_paths.py`, `heal_bridges_osm.py`, `mitigate_branch_outlet_backpool.py`, `split_flows.py`, `stream_branches.py`, `usgs_gage_unit_setup.py`: Fix backwards incompatibilities with new version of `geopandas`.
+
+<br/><br/>
+
 ## v4.5.5.1 - 2024-08-16 - [PR#1225](https://github.com/NOAA-OWP/inundation-mapping/pull/1225)
 
 Removes warning when running `heal_bridges_osm.py` by not saving the empty DataFrame.

src/build_stream_traversal.py

@@ -64,9 +64,7 @@ def execute(self, streams, wbd8, hydro_id):
                     {'geometry': stream_midpoint}, crs=streams.crs, geometry='geometry'
                 )
                 stream_wbdjoin = gpd.sjoin(stream_md_gpd, wbd8, how='left', predicate='within')
-                stream_wbdjoin = stream_wbdjoin.rename(
-                    columns={"geometry": "midpoint", "index_right": "HUC8id"}
-                )
+                stream_wbdjoin = stream_wbdjoin.rename(columns={"geometry": "midpoint", "fimid": "HUC8id"})
                 streams = streams.join(stream_wbdjoin).drop(columns=['midpoint'])
 
                 streams['seqID'] = (

src/derive_level_paths.py

@@ -192,7 +192,10 @@ def Derive_level_paths(
         )
 
         stream_network = stream_network.remove_branches_in_waterbodies(
-            waterbodies=waterbodies, out_vector_files=out_stream_network_dissolved, verbose=False
+            waterbodies=waterbodies,
+            out_vector_files=out_stream_network_dissolved,
+            branch_id_attribute=branch_id_attribute,
+            verbose=False,
         )
         stream_network = stream_network.select_branches_intersecting_huc(
             wbd=wbd,

src/heal_bridges_osm.py

@@ -74,9 +74,9 @@ def process_bridges_in_huc(
     # Check if the GeoDataFrame is empty
     if not osm_gdf.empty:
         # Write the bridge points to a geopackage
-        osm_gdf.to_file(bridge_centroids, index=False)
+        osm_gdf.to_file(bridge_centroids, index=False, engine='fiona')
     else:
-        print('The geoDataFrame is empty.File not saved.')
+        print('The geoDataFrame is empty. File not saved.')
     return
 
 

src/mitigate_branch_outlet_backpool.py

@@ -390,7 +390,7 @@ def mask_raster_to_boundary(raster_path, boundary_json, save_path):
                     trimmed_flows, inital_length_km = snap_and_trim_splitflow(pt_3tl_geom, split_flows_geom)
 
                     # Create buffer around the updated flows geodataframe (and make sure it's all one shape)
-                    buffer = trimmed_flows.buffer(10).geometry.unary_union
+                    buffer = trimmed_flows.buffer(10).geometry.union_all()
 
                     # Remove flowpoints that don't intersect with the trimmed flow line
                     split_points_filtered_geom = split_points_geom[split_points_geom.geometry.within(buffer)]

src/split_flows.py

@@ -370,7 +370,7 @@ def snap_and_trim_flow(snapped_point, flows):
         split_flows_gdf = gpd.sjoin(
             split_flows_gdf, lakes_buffer, how='left', predicate='within'
         )  # Note: Options include intersects, within, contains, crosses
-        split_flows_gdf = split_flows_gdf.rename(columns={"index_right": "LakeID"}).fillna(-999)
+        split_flows_gdf = split_flows_gdf.rename(columns={"newID": "LakeID"}).fillna(-999)
     else:
         split_flows_gdf['LakeID'] = -999
 

src/stream_branches.py

@@ -188,6 +188,22 @@ def drop(self, labels=None, axis=0):
         )
         return self
 
+    def rename(self, columns):
+        branch_id_attribute = self.branch_id_attribute
+        attribute_excluded = self.attribute_excluded
+        values_excluded = self.values_excluded
+
+        self = super(gpd.GeoDataFrame, self)
+        self = self.rename(columns=columns)
+
+        self = StreamNetwork(
+            self,
+            branch_id_attribute=branch_id_attribute,
+            attribute_excluded=attribute_excluded,
+            values_excluded=values_excluded,
+        )
+        return self
+
     def dissolve(self, by=None):
         branch_id_attribute = self.branch_id_attribute
         attribute_excluded = self.attribute_excluded
@@ -645,7 +661,9 @@ def find_upstream_reaches_in_waterbodies(tmp_self, tmp_IDs=[]):
 
         return self
 
-    def remove_branches_in_waterbodies(self, waterbodies, out_vector_files=None, verbose=False):
+    def remove_branches_in_waterbodies(
+        self, waterbodies, branch_id_attribute, out_vector_files=None, verbose=False
+    ):
         """
         Removes branches completely in waterbodies
         """
@@ -658,9 +676,13 @@ def remove_branches_in_waterbodies(self, waterbodies, out_vector_files=None, ver
             waterbodies = gpd.read_file(waterbodies)
 
         if isinstance(waterbodies, gpd.GeoDataFrame):
+            waterbodies = waterbodies.drop('OBJECTID', axis=1)
+
             # Find branches in waterbodies
+            self = self.rename(columns={branch_id_attribute: "bids"})
             sjoined = gpd.sjoin(self, waterbodies, predicate="within")
             self = self.drop(sjoined.index)
+            self = self.rename(columns={"bids": branch_id_attribute})
 
             if out_vector_files is not None:
                 if verbose:
@@ -693,7 +715,9 @@ def select_branches_intersecting_huc(self, wbd, buffer_wbd_streams, out_vector_f
 
         if isinstance(wbd, gpd.GeoDataFrame):
             # Find branches intersecting HUC
+            self = self.rename(columns={branch_id_attribute: "bids"})
             sjoined = gpd.sjoin(self, wbd, predicate="intersects")
+            self = self.rename(columns={"bids": branch_id_attribute})
 
             self = self[self.index.isin(sjoined.index.values)]
 

src/usgs_gage_unit_setup.py

@@ -94,7 +94,7 @@ def sort_into_branch(self, nwm_subset_streams_levelPaths):
 
         if not self.gages[self.gages.feature_id.isnull()].empty:
             missing_feature_id = self.gages.loc[self.gages.feature_id.isnull()].copy()
-            nwm_reaches_union = nwm_reaches.geometry.unary_union
+            nwm_reaches_union = nwm_reaches.geometry.union_all()
             missing_feature_id['feature_id'] = missing_feature_id.apply(
                 lambda row: self.sjoin_nearest_to_nwm(row.geometry, nwm_reaches, nwm_reaches_union), axis=1
             )
@@ -118,7 +118,7 @@ def branch_zero(self, bzero_id):
         return self.gages
 
     def write(self, out_name):
-        self.gages.to_file(out_name, driver='GPKG', index=False)
+        self.gages.to_file(out_name, driver='GPKG', index=False, engine='fiona')
 
     @staticmethod
     def sjoin_nearest_to_nwm(pnt, lines, union):