Merge pull request #34 from addelany/main

Update config files with new qaqc branch

configuration/defaultV2/observation_processing.yml

@@ -20,7 +20,7 @@ sss_fname: fcre-manual-data/FCR_SSS_inflow_2013_2020.csv
 noaa_location: FCRE
 
 met_raw_obs_fname:
-    - fcre-metstation-data/FCRmet_L1.csv
+    - fcre-metstation-data-qaqc/FCRmet_L1.csv
     - Met_final_2015_2022.csv
 #specified_metfile: .na #NA #paste0(manual_data_location, "/met_full_postQAQC.csv")
 
@@ -34,7 +34,7 @@ nldas: fcre-manual-data/FCR_GLM_NLDAS_010113_123119_GMTadjusted.csv
 
 #specified_inflow1: .na #manual-data/FCR_weir_inflow_2013_2019_20200624_allfractions_2poolsDOC.csv
 inflow_raw_file1:
-    - fcre-weir-data/FCRWeir_L1.csv
+    - fcre-weir-data-qaqc/FCRWeir_L1.csv
     - inflow_for_EDI_2013_10Jan2021.csv
 #specified_inflow2: .na #manual-data/FCR_wetland_inflow_2013_2019_20200713_allfractions_2DOCpools.csv
 #specified_outflow1: .na #manual-data/FCR_spillway_outflow_SUMMED_WeirWetland_2013_2019_20200615.csv
@@ -89,16 +89,16 @@ depths_bins_top:
 
 obs_config_file: observations_config_ch4_dic.csv
 
-realtime_insitu_location: fcre-catwalk-data
-realtime_met_station_location: fcre-metstation-data
+realtime_insitu_location: fcre-catwalk-data-qaqc
+realtime_met_station_location: fcre-metstation-data-qaqc
 manual_data_location: fcre-manual-data
-realtime_inflow_data_location: fcre-weir-data
+realtime_inflow_data_location: fcre-weir-data-qaqc
 
 specified_inflow1: fcre-manual-data/FCR_weir_inflow_2013_2019_20200624_allfractions_2poolsDOC.csv
 specified_outflow1: fcre-manual-data/FCR_spillway_outflow_SUMMED_WeirWetland_2013_2019_20200615.csv
 
 insitu_obs_fname:
-    - fcre-catwalk-data/fcre-waterquality_L1.csv
+    - fcre-catwalk-data-qaqc/fcre-waterquality_L1.csv
     - FCR_Catwalk_2018_2022.csv
 secchi_fname: Secchi_depth_2013-2020.csv
 ctd_fname: CTD_final_2013_2021.csv
@@ -114,7 +114,7 @@ ctd_2_exo_sensor_chla: [0, 1]  #c(-2.0430, 2.5314) #c(-1.582, 1.335)
 ctd_2_exo_sensor_do: [0, 1] #c(-10.770, 1.061)
 exo_sensor_2_grab_sample_fdom: [-151.3407, 29.62654]
 focal_depths: .na #1.6
-maintenance_file: fcre-catwalk-data/CAT_MaintenanceLog.txt
+maintenance_file: fcre-catwalk-data-qaqc/CAT_MaintenanceLog.txt
 
 pull_from_git: TRUE
 

configuration/ltreb/configure_flare_glm_aed.yml

@@ -9,8 +9,7 @@ file_path:
 s3:
   drivers:
     endpoint: s3.flare-forecast.org
-    #bucket: drivers/noaa/clim-annual
-    bucket: drivers/noaa/gefs-v12-reprocess
+    bucket: drivers/noaa/clim-annual
   inflow_drivers:
     endpoint: s3.flare-forecast.org
     bucket: drivers/testing/inflow/parquet
@@ -57,8 +56,8 @@ metadata:
 da_setup:
   da_method: enkf
   par_fit_method: perturb_const
-  ensemble_size: 100
-  localization_distance: 1
+  ensemble_size: 10
+  localization_distance: .na
   no_negative_states: yes
   assimilate_first_step: no
   use_obs_constraint: TRUE

workflows/ltreb/combined_run_aed.R

@@ -13,7 +13,7 @@ use_s3 <- FALSE
 files.sources <- list.files(file.path(lake_directory, "R"), full.names = TRUE)
 sapply(files.sources, source)
 
-sim_names <- "ltreb_local_one"
+sim_names <- "ltreb"
 config_set_name <- "ltreb"
 
 config_files <- paste0("configure_flare_glm_aed.yml")
@@ -22,8 +22,7 @@ num_forecasts <- 2
 #num_forecasts <- 1 #52 * 3 - 3
 #num_forecasts <- 1#19 * 7 + 1
 days_between_forecasts <- 0
-#forecast_horizon <- 30 #364 #32
-forecast_horizon <- 0 #364 #32
+forecast_horizon <- 30 #364 #32
 starting_date <- as_date("2021-01-01")
 #second_date <- as_date("2020-12-01") - days(days_between_forecasts)
 second_date <- as_date("2021-12-31") #- days(days_between_forecasts)
@@ -34,13 +33,6 @@ second_date <- as_date("2021-12-31") #- days(days_between_forecasts)
 
 #second_date <- as_date("2018-08-01") - days(days_between_forecasts)
 
-
-daily <- seq.Date(starting_date,second_date, by = 1)
-
-fortnightly <- daily[seq(1,length(daily),by=14)]
-
-
-
 ## OLD CODE
 start_dates <- rep(NA, num_forecasts)
 start_dates[1:2] <- c(starting_date, second_date)
@@ -216,10 +208,10 @@ if(starting_index == 1){
 #   arrange(start_dates)
 #
 # sims$horizon[1:length(models)] <- 0
-#
 
-for(i in 1:1){
-#for(i in starting_index:length(forecast_start_dates)){
+
+#for(i in 1:1){
+for(i in starting_index:length(forecast_start_dates)){
 
   #https_file <- "https://raw.githubusercontent.com/cayelan/FCR-GLM-AED-Forecasting/master/FCR_2013_2019GLMHistoricalRun_GLMv3beta/inputs/FCR_SSS_inflow_2013_2021_20211102_allfractions_2DOCpools.csv"
   #if(!file.exists(file.path(config$file_path$execute_directory, basename(https_file)))){
@@ -287,9 +279,6 @@ for(i in 1:1){
                                               use_forecast = config$met$use_forecasted_met,
                                               use_siteid_s3 = FALSE)
 
-  ## manipulate code to only include one met ensemble member - REMOVE LATER
-  #met_out$filenames <- met_out$filenames[1]
-
   if(config$model_settings$model_name == "glm_aed"){
     variables <- c("time", "FLOW", "TEMP", "SALT",
                    'OXY_oxy',
@@ -346,15 +335,6 @@ for(i in 1:1){
                                    obs_config = obs_config,
                                    config)
 
-
-  full_time <- seq(lubridate::as_datetime(config$run_config$start_datetime), lubridate::as_datetime(config$run_config$forecast_start_datetime) + lubridate::days(config$run_config$forecast_horizon), by = "1 day")
-  full_time <- as.Date(full_time)
-
-  idx <- which(!full_time %in% fortnightly)
-  obs[, idx, ] <- NA
-
-
-
   obs_secchi_depth <- get_obs_secchi_depth(obs_file = file.path(config$file_path$qaqc_data_directory,paste0(config$location$site_id, "-targets-insitu.csv")),
                                            start_datetime = config$run_config$start_datetime,
                                            end_datetime = config$run_config$end_datetime,