Turn on EDEN updates [minor]

DataCleaningScripts/download_eden.R

@@ -79,15 +79,15 @@ get_files_to_update <- function(eden_path = file.path("Water"),
   # Find files that have been updated since last download
   last_download <- get_last_download(eden_path, metadata, force_update = force_update)
   new <- metadata %>%
-    dplyr::left_join(last_download, by = "dataset", suffix = c(".curr", ".last")) %>%
-    dplyr::filter(last_modified.curr > last_modified.last | size.curr != size.last | is.na(last_modified.last))
+    dplyr::left_join(last_download, by = "dataset", suffix = c("", ".last")) %>%
+    dplyr::filter(last_modified > last_modified.last | size != size.last | is.na(last_modified.last))
   metadata %>%
     dplyr::filter(year %in% c(new$year-2, new$year-1, new$year, new$year+1, new$year+2))
   }
 
 #' @name update_last_download
 #'
-#' @title Write new metata file for files already downloaded
+#' @title Write new metadata file for files already downloaded
 #'
 #' @param eden_path path where the EDEN data should be stored
 #' @param metadata EDEN file metadata

DataCleaningScripts/eden_covariates.R

@@ -79,6 +79,7 @@ calc_reversals <- function(depth_data) {
                                               is.na(depth) ~ units::set_units(NA, d)),
                   .keep = "none") %>%
     stars::st_apply(c(1, 2), sum)
+  return(reversals)
 }
 
 
@@ -97,8 +98,16 @@ extract_region_means <- function(raster, regions) {
   var_name <- names(raster)
   region_means <- raster::aggregate(raster, regions, mean, na.rm=TRUE) %>%
     setNames(., "value")
+  if(all(is.nan(region_means$value))) {
+    region_means_spdf <- regions %>%
+      dplyr::mutate(variable = var_name, value = NA)
+  } else {
   region_means_spdf <- regions %>%
-    dplyr::mutate(variable = var_name, value = region_means$value)
+    dplyr::mutate(variable = var_name, value = as.numeric(region_means$value)) %>%
+    dplyr::mutate_if(is.numeric, list(~dplyr::na_if(., Inf))) %>%
+    dplyr::mutate_if(is.numeric, list(~dplyr::na_if(., -Inf))) %>%
+    dplyr::mutate_if(is.numeric, list(~dplyr::na_if(., NaN))) 
+  }
   return(region_means_spdf)
 }
 
@@ -115,10 +124,18 @@ extract_region_means <- function(raster, regions) {
 
 available_years <- function(eden_path = file.path("Water")) {
   eden_data_files <- list.files(file.path(eden_path), pattern = '_depth.nc')
+
+  # Find which years need to be updated since last download
+  metadata <- get_metadata()
+  last_download <- get_last_download(eden_path, metadata)
+  new <- metadata %>%
+    dplyr::left_join(last_download, by = "dataset", suffix = c("", ".last")) %>%
+    dplyr::filter(last_modified > last_modified.last | size != size.last | is.na(last_modified.last))
   years <- eden_data_files %>%
     stringr::str_split('_', simplify = TRUE) %>%
     .[, 1] %>%
-    unique()
+    unique() %>%
+    .[. %in% c(new$year, new$year+1, new$year+2)]
   return(years)
 }
 
@@ -153,7 +170,11 @@ get_eden_covariates <- function(level = "subregions",
   for (year in years) {
     print(paste("Processing ", year, "...", sep = ""))
     pattern <- file.path(paste(year, "_.*_depth.nc", sep = ''))
-    nc_files <- list.files(eden_path, pattern, full.names = TRUE)
+    pattern2 <- file.path(paste(as.numeric(year)-1, "_.*_depth.nc", sep = ''))
+    pattern3 <- file.path(paste(as.numeric(year)-2, "_.*_depth.nc", sep = ''))
+    nc_files <- c(list.files(eden_path, pattern3, full.names = TRUE),
+                  list.files(eden_path, pattern2, full.names = TRUE),
+                  list.files(eden_path, pattern, full.names = TRUE))
     year_data <- stars::read_stars(nc_files, along = "time") %>%
       setNames(., "depth") %>%
       dplyr::mutate(depth = dplyr::case_when(depth < units::set_units(0, cm) ~ units::set_units(0, cm),
@@ -164,6 +185,11 @@ get_eden_covariates <- function(level = "subregions",
     breed_end <- as.POSIXct(paste0(year, '-06-30'))
     breed_season_data <- year_data %>%
       dplyr::filter(time >= breed_start, time <= breed_end)
+
+    dry_start <- as.POSIXct(paste0(as.numeric(year)-2, '-03-31'))
+    dry_end <- as.POSIXct(paste0(year, '-06-30'))
+    dry_season_data <- year_data %>%
+      dplyr::filter(time >= dry_start, time <= dry_end)
 
     # Do a pre-breed/post-breed split to allow pre-breeding recession calculations
     # following Peterson 2017. Peterson does this on a per species basis. To start
@@ -189,14 +215,8 @@ get_eden_covariates <- function(level = "subregions",
     post_recession <- calc_recession(post_breed_season_data) %>%
       setNames(., "post_recession")
 
-    # Calculate dryindex from everwader
-    # TODO: USGS code calculates this from t-3 3/31 to t 6/30 and converts
-    # the first value to NA
-    # To replicate we would need to load three years worth of data and be careful with
-    # other applications using min/max or replace this with a different predictor
-    # since it's unclear why the lag would be important here
-    # we could also fit the lag rather than building it into the feature
-    dry_days <- calc_dry_days(breed_season_data) %>%
+    # Calculate dryindex from everwader (USGS code calculates this from t-3 3/31 to t 6/30)
+    dry_days <- calc_dry_days(dry_season_data) %>%
       setNames(., "dry_days")
 
     # Calculate reversals following Peterson 2017

DataCleaningScripts/get_water_data.R

@@ -12,24 +12,24 @@ get_eden_data <- function() {
 download_eden_depths()
 
 covariate_data <- read.table("Water/eden_covariates.csv", header = TRUE, sep = ",")
-new_data <- get_eden_covariates()
-new_data2 <- get_eden_covariates(level="all")
-new_data3 <- get_eden_covariates(level="wcas")
-all_data <- dplyr::bind_rows(new_data,new_data2,new_data3) %>%
-  dplyr::select(year, region=Name, variable, value) %>%
-  as.data.frame() %>%
-  dplyr::select(-geometry) %>%
-  tidyr::pivot_wider(names_from="variable", values_from="value") %>%
-  dplyr::arrange("year", "region")
-new_covariates <- all_data %>%
-  merge(dplyr::filter(covariate_data, !date %in% all_data$year)) %>%
-  dplyr::arrange("year", "region")
+new_covariates <- get_eden_covariates(eden_path = file.path("Water")) %>%
+                  dplyr::bind_rows(get_eden_covariates(eden_path = file.path("Water"), level="all")) %>%
+                  dplyr::bind_rows(get_eden_covariates(eden_path = file.path("Water"), level="wcas")) %>%
+                  dplyr::select(year, region=Name, variable, value) %>%
+                  as.data.frame() %>%
+                  dplyr::select(-geometry) %>%
+                  tidyr::pivot_wider(names_from="variable", values_from="value") %>%
+                  dplyr::mutate(year = as.integer(year)) %>%
+                  dplyr::arrange("year", "region")
+covariate_data <- dplyr::filter(covariate_data, !year %in% new_covariates$year) %>%
+                  rbind(new_covariates) %>%
+                  dplyr::arrange("year", "region")
 
 depth_data <- read.table("Water/eden_depth.csv", header = TRUE, sep = ",") %>%
               dplyr::mutate(date=as.Date(date))
-new_depths <- get_eden_depths() %>%
-              dplyr::bind_rows(get_eden_depths(level="all")) %>%
-              dplyr::bind_rows(get_eden_depths(level="wcas")) %>%
+new_depths <- get_eden_depths(eden_path = file.path("Water")) %>%
+              dplyr::bind_rows(get_eden_depths(eden_path = file.path("Water"), level="all")) %>%
+              dplyr::bind_rows(get_eden_depths(eden_path = file.path("Water"), level="wcas")) %>%
               dplyr::mutate(date=as.Date(date))
 
 depth_data <- dplyr::filter(depth_data, !date %in% new_depths$date) %>%
@@ -38,7 +38,7 @@ depth_data <- dplyr::filter(depth_data, !date %in% new_depths$date) %>%
 
 file.remove(dir(path=file.path('Water'),  pattern="_.*_depth.nc"))
 
-return(list(new_covariates=new_covariates, depth_data=depth_data))
+return(list(covariate_data=covariate_data, depth_data=depth_data))
 }
 
 #' Writes new water data
@@ -49,7 +49,7 @@ update_water <- function() {
 
   data <- get_eden_data()
 
-  write.table(data$new_covariates, "Water/eden_covariates.csv", row.names = FALSE, col.names = TRUE,
+  write.table(data$covariate_data, "Water/eden_covariates.csv", row.names = FALSE, col.names = TRUE,
             na="", sep = ",", quote = FALSE)
 
   write.table(data$depth_data, file = "Water/eden_depth.csv",