From a3ce087eeb0ab408e57633ea20db5196e5461992 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 5 Mar 2024 18:10:41 -0600 Subject: [PATCH 01/22] Update package description --- DESCRIPTION | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index c5be3794d..4e22b5e8c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -29,24 +29,15 @@ Authors@R: c( email = "sebastian.funk@lshtm.ac.uk", role = c("aut"))) Description: - Provides a collection of metrics and proper scoring rules - (Tilmann Gneiting & Adrian E Raftery (2007) - , Jordan, A., Krüger, F., & Lerch, S. (2019) - ) within a consistent framework for - evaluation, comparison and visualisation of forecasts. - In addition to proper scoring rules, functions are provided to assess - bias, sharpness and calibration - (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind - M. Eggo, W. John Edmunds (2019) ) of - forecasts. - Several types of predictions (e.g. binary, discrete, continuous) which may - come in different formats (e.g. forecasts represented by predictive samples - or by quantiles of the predictive distribution) can be evaluated. - Scoring metrics can be used either through a convenient data.frame format, - or can be applied as individual functions in a vector / matrix format. - All functionality has been implemented with a focus on performance and is - robustly tested. Find more information about the package in the - accompanying paper (). + `scoringutils` facilitates the evaluation of forecasts in a convenient + framework based on `data.table`. It allows user to to check their forecasts + and diagnose issues, to visualise forecasts and missing data, to transform + data before scoring, to handle missing forecasts, to aggregate scores, and + to visualise the results of the evaluation. The package mostly focuses on + the evaluation of probabilistic forecasts and allows evaluating several + different forecast types and input formats. Find more information about the + package in the Vignettes as well as in the accompanying paper + (). License: MIT + file LICENSE Encoding: UTF-8 LazyData: true From 20593fd361d4450ea3304d0e57cd092868f16225 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 6 Mar 2024 00:14:28 +0000 Subject: [PATCH 02/22] Automatic readme update [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba7e3c6e9..4b1e5afe9 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ example_quantile %>% #> underprediction dispersion bias interval_coverage_50 #> #> 1: 4237.177310 3663.52458 -0.05640625 0.3906250 -#> 2: 10284.972826 4102.50094 0.09726563 0.3281250 +#> 2: 10284.972826 4102.50094 0.09726562 0.3281250 #> 3: 3260.355639 5664.37795 -0.07890625 0.4687500 #> 4: 4.103261 30.18099 0.07265625 0.8750000 #> 5: 2.098505 91.40625 0.33906250 0.6640625 From d88c4d41cfd03b97b0530decf2e0c74e62b20273 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 5 Mar 2024 18:19:10 -0600 Subject: [PATCH 03/22] Update documentation for `get_coverage()` --- R/add_coverage.R | 25 +++++++++++-------------- man/get_coverage.Rd | 27 ++++++++++++++------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/R/add_coverage.R b/R/add_coverage.R index 7ee239e11..cb0117b61 100644 --- a/R/add_coverage.R +++ b/R/add_coverage.R @@ -1,9 +1,14 @@ #' @title Get Quantile And Interval Coverage Values For Quantile-Based Forecasts #' -#' @description Compute interval coverage of central prediction intervals, -#' quantile coverage for predictive quantiles, as well as the deviation between -#' desired and actual coverage to a data.table. Forecasts should be in a -#' quantile format (following the input requirements of `score()`). +#' @description For a validated forecast object in a quantile-based format +#' (see [as_forecast()] for more information), this function computes +#' - interval coverage of central prediction intervals +#' - quantile coverage for predictive quantiles +#' - the deviation between desired and actual coverage (both for interval and +#' quantile coverage) +#' +#' Coverage values are computed for a specific level of grouping, as specified +#' in the `by` argument. By default, coverage values are computed per model. #' #' **Interval coverage** #' @@ -14,15 +19,6 @@ #' central prediction interval is the interval between the 0.25 and 0.75 #' quantiles of the predictive distribution. #' -#' The function `get_coverage()` computes the coverage per central prediction -#' interval. This means that if you set `by` to the unit of a single forecast, -#' interval coverage will always be either `TRUE` -#' (observed value falls within the interval) or `FALSE` (observed value falls -#' outside the interval) and analogously for quantile coverage. -#' Coverage values become meaningful by summarising them across different -#' dimensions, as specified in the `by` argument (thereby returning the -#' proportion of values covered by all prediction intervals/quantiles). -#' #' **Quantile coverage** #' #' Quantile coverage for a given quantile is defined as the proportion of @@ -38,7 +34,8 @@ #' (can be either interval or quantile coverage) and the #' actual coverage. For example, if the desired coverage is 90% and the actual #' coverage is 80%, the coverage deviation is -0.1. -#' +#' @return A data.table with columns as specified in `by` and additional +#' columns for the coverage values described above #' @inheritParams score #' @param by character vector that denotes the level of grouping for which the #' coverage values should be computed. By default (`"model"`), one coverage diff --git a/man/get_coverage.Rd b/man/get_coverage.Rd index eea37bf44..e4ac7a7a8 100644 --- a/man/get_coverage.Rd +++ b/man/get_coverage.Rd @@ -14,15 +14,25 @@ coverage values should be computed. By default (\code{"model"}), one coverage value per model will be returned.} } \value{ +A data.table with columns as specified in \code{by} and additional +columns for the coverage values described above + a data.table with columns "interval_coverage", "interval_coverage_deviation", "quantile_coverage", "quantile_coverage_deviation" and the columns specified in \code{by}. } \description{ -Compute interval coverage of central prediction intervals, -quantile coverage for predictive quantiles, as well as the deviation between -desired and actual coverage to a data.table. Forecasts should be in a -quantile format (following the input requirements of \code{score()}). +For a validated forecast object in a quantile-based format +(see \code{\link[=as_forecast]{as_forecast()}} for more information), this function computes +\itemize{ +\item interval coverage of central prediction intervals +\item quantile coverage for predictive quantiles +\item the deviation between desired and actual coverage (both for interval and +quantile coverage) +} + +Coverage values are computed for a specific level of grouping, as specified +in the \code{by} argument. By default, coverage values are computed per model. \strong{Interval coverage} @@ -33,15 +43,6 @@ by two quantiles that denote the lower and upper bound. For example, the 50\% central prediction interval is the interval between the 0.25 and 0.75 quantiles of the predictive distribution. -The function \code{get_coverage()} computes the coverage per central prediction -interval. This means that if you set \code{by} to the unit of a single forecast, -interval coverage will always be either \code{TRUE} -(observed value falls within the interval) or \code{FALSE} (observed value falls -outside the interval) and analogously for quantile coverage. -Coverage values become meaningful by summarising them across different -dimensions, as specified in the \code{by} argument (thereby returning the -proportion of values covered by all prediction intervals/quantiles). - \strong{Quantile coverage} Quantile coverage for a given quantile is defined as the proportion of From f5384185475f146f4f743c72fed354aa68eced30 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 5 Mar 2024 18:21:05 -0600 Subject: [PATCH 04/22] Update documentation for get_forecast_counts() --- R/available_forecasts.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/R/available_forecasts.R b/R/available_forecasts.R index 372068bdd..0411be9cb 100644 --- a/R/available_forecasts.R +++ b/R/available_forecasts.R @@ -1,10 +1,10 @@ #' @title Count Number of Available Forecasts #' #' @description -#' -#' Given a data set with forecasts, count the number of available forecasts -#' for arbitrary grouping (e.g. the number of forecasts per model, or the -#' number of forecasts per model and location). +#' Given a data set with forecasts, count the number of available forecasts. +#' The level of grouping can be specified using the `by` argument (e.g. to +#' count the number of forecasts per model, or the number of forecasts per +#' model and location). #' This is useful to determine whether there are any missing forecasts. #' #' @param by character vector or `NULL` (the default) that denotes the From b701153f65c67afb418ab0f80e332e8f3011e20d Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Mar 2024 10:33:51 -0600 Subject: [PATCH 05/22] update docs --- man/get_forecast_counts.Rd | 7 ++++--- man/scoringutils-package.Rd | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/man/get_forecast_counts.Rd b/man/get_forecast_counts.Rd index ecb50b991..50bb77b27 100644 --- a/man/get_forecast_counts.Rd +++ b/man/get_forecast_counts.Rd @@ -33,9 +33,10 @@ A data.table with columns as specified in \code{by} and an additional column "count" with the number of forecasts. } \description{ -Given a data set with forecasts, count the number of available forecasts -for arbitrary grouping (e.g. the number of forecasts per model, or the -number of forecasts per model and location). +Given a data set with forecasts, count the number of available forecasts. +The level of grouping can be specified using the \code{by} argument (e.g. to +count the number of forecasts per model, or the number of forecasts per +model and location). This is useful to determine whether there are any missing forecasts. } \examples{ diff --git a/man/scoringutils-package.Rd b/man/scoringutils-package.Rd index e57db037b..c41610937 100644 --- a/man/scoringutils-package.Rd +++ b/man/scoringutils-package.Rd @@ -5,9 +5,9 @@ \alias{scoringutils-package} \title{scoringutils: Utilities for Scoring and Assessing Predictions} \description{ -Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about the package in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). +`scoringutils` facilitates the evaluation of forecasts in a convenient framework based on `data.table`. It allows user to to check their forecasts and diagnose issues, to visualise forecasts and missing data, to transform data before scoring, to handle missing forecasts, to aggregate scores, and to visualise the results of the evaluation. The package mostly focuses on the evaluation of probabilistic forecasts and allows evaluating several different forecast types and input formats. Find more information about the package in the Vignettes as well as in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). -Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about the package in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). +`scoringutils` facilitates the evaluation of forecasts in a convenient framework based on `data.table`. It allows user to to check their forecasts and diagnose issues, to visualise forecasts and missing data, to transform data before scoring, to handle missing forecasts, to aggregate scores, and to visualise the results of the evaluation. The package mostly focuses on the evaluation of probabilistic forecasts and allows evaluating several different forecast types and input formats. Find more information about the package in the Vignettes as well as in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). } \seealso{ Useful links: From d73a6edd9c4c1a0606c3000a3e78e63c48194035 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Mar 2024 23:33:21 -0600 Subject: [PATCH 06/22] Update docs for `transform_forecasts()` and input checks for `log_shift()` --- R/convenience-functions.R | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/R/convenience-functions.R b/R/convenience-functions.R index 3be8bc071..ea74796e2 100644 --- a/R/convenience-functions.R +++ b/R/convenience-functions.R @@ -44,11 +44,11 @@ #' @param ... Additional parameters to pass to the function you supplied. For #' the default option of [log_shift()] this could be the `offset` argument. #' -#' @return A `data.table` with either a transformed version of the data, or one -#' with both the untransformed and the transformed data. includes the original -#' data as well as a transformation of the original data. There will be one -#' additional column, `scale', present which will be set to "natural" for the -#' untransformed forecasts. +#' @return A forecast object with either a transformed version of the data, or +#' one with both the untransformed and the transformed data. includes the +#' original data as well as a transformation of the original data. There will +#' be one additional column, `scale', present which will be set to "natural" +#' for the untransformed forecasts. #' #' @importFrom data.table ':=' is.data.table copy #' @author Nikos Bosse \email{nikosbosse@@gmail.com} @@ -179,6 +179,7 @@ transform_forecasts <- function(data, #' \doi{https://doi.org/10.1101/2023.01.23.23284722} #' # nolint #' @keywords check-forecasts +#' @importFrom checkmate assert_numeric assert_number #' @examples #' log_shift(1:10) #' log_shift(0:9, offset = 1) @@ -191,6 +192,10 @@ transform_forecasts <- function(data, log_shift <- function(x, offset = 0, base = exp(1)) { + assert_numeric(x, min.len = 1) + assert_number(offset) + assert_number(base) + if (any(x < 0, na.rm = TRUE)) { w <- paste("Detected input values < 0.") stop(w) From 30d0da75f3ce08acd32340ef48a2761e9e432711 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Mar 2024 23:37:31 -0600 Subject: [PATCH 07/22] delete docs template for scoring input data --- R/documentation-templates.R | 8 -------- 1 file changed, 8 deletions(-) diff --git a/R/documentation-templates.R b/R/documentation-templates.R index 96afea22a..4959e8f2d 100644 --- a/R/documentation-templates.R +++ b/R/documentation-templates.R @@ -84,11 +84,3 @@ NULL #' @name document_test_functions #' @keywords internal NULL - -#' Documentation template for scoring input data -#' @param data A data frame (or similar) of forecasts following the -#' specifications detailed in [score()]. -#' @param scores A data.table of scores as produced by [score()]. -#' @name document_score_data -#' @keywords internal -NULL From 929b996f3765b2c4bf635ec86b871b4bd702daac Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Mar 2024 23:50:05 -0600 Subject: [PATCH 08/22] update `get_`-functions --- R/get_-functions.R | 44 ++++++++++++++------------------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/R/get_-functions.R b/R/get_-functions.R index f5677403e..a6390cea1 100644 --- a/R/get_-functions.R +++ b/R/get_-functions.R @@ -170,7 +170,7 @@ get_score_names <- function(scores, error = FALSE) { } -#' @title Get unit of a single forecast +#' @title Get Unit Of A Single Forecast #' @description Helper function to get the unit of a single forecast, i.e. #' the column names that define where a single forecast was made for. #' This just takes all columns that are available in the data and subtracts @@ -180,9 +180,11 @@ get_score_names <- function(scores, error = FALSE) { #' @inheritParams validate_forecast #' @return A character vector with the column names that define the unit of #' a single forecast +#' @importFrom checkmate assert_data_frame #' @export #' @keywords check-forecasts get_forecast_unit <- function(data) { + assert_data_frame(data) protected_columns <- get_protected_columns(data) protected_columns <- c(protected_columns, attr(data, "score_names")) forecast_unit <- setdiff(colnames(data), unique(protected_columns)) @@ -190,7 +192,7 @@ get_forecast_unit <- function(data) { } -#' @title Get protected columns from a data frame +#' @title Get Protected Columns From Data #' #' @description Helper function to get the names of all columns in a data frame #' that are protected columns. @@ -229,7 +231,7 @@ get_protected_columns <- function(data = NULL) { } -#' @title Find duplicate forecasts +#' @title Find Duplicate Forecasts #' #' @description Helper function to identify duplicate forecasts, i.e. #' instances where there is more than one forecast for the same prediction @@ -238,17 +240,23 @@ get_protected_columns <- function(data = NULL) { #' @param data A data.frame as used for [score()] #' #' @param forecast_unit A character vector with the column names that define -#' the unit of a single forecast. If `NULL` (the default) the function tries -#' to infer the unit of a single forecast. +#' the unit of a single forecast. By default the forecast unit will be +#' automatically inferred from the data (see [get_forecast_unit()]) #' #' @return A data.frame with all rows for which a duplicate forecast was found #' @export +#' @importFrom checkmate assert_data_frame assert_subset #' @keywords check-forecasts #' @examples #' example <- rbind(example_quantile, example_quantile[1000:1010]) #' get_duplicate_forecasts(example) -get_duplicate_forecasts <- function(data, forecast_unit = NULL) { +get_duplicate_forecasts <- function( + data, + forecast_unit = get_forecast_unit(data) +) { + assert_data_frame(data) + assert_subset(forecast_unit, colnames(data)) available_type <- c("sample_id", "quantile_level") %in% colnames(data) type <- c("sample_id", "quantile_level")[available_type] if (is.null(forecast_unit)) { @@ -260,27 +268,3 @@ get_duplicate_forecasts <- function(data, forecast_unit = NULL) { out[, scoringutils_InternalDuplicateCheck := NULL] return(out[]) } - - -#' @title Get a list of all attributes of a scoringutils object -#' -#' @param object A object of class `forecast_` -#' -#' @return A named list with the attributes of that object. -#' @keywords internal -get_scoringutils_attributes <- function(object) { - possible_attributes <- c( - "scoringutils_by", - "forecast_unit", - "forecast_type", - "score_names", - "messages", - "warnings" - ) - - attr_list <- list() - for (attr_name in possible_attributes) { - attr_list[[attr_name]] <- attr(object, attr_name) - } - return(attr_list) -} From 7e526b0b6554dd3a0b9c323364fd75e5dc94d98f Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Mar 2024 23:51:11 -0600 Subject: [PATCH 09/22] Update docs --- man/check_duplicates.Rd | 4 ++-- man/document_score_data.Rd | 15 --------------- man/get_duplicate_forecasts.Rd | 8 ++++---- man/get_forecast_unit.Rd | 2 +- man/get_protected_columns.Rd | 2 +- man/get_scoringutils_attributes.Rd | 18 ------------------ man/transform_forecasts.Rd | 10 +++++----- 7 files changed, 13 insertions(+), 46 deletions(-) delete mode 100644 man/document_score_data.Rd delete mode 100644 man/get_scoringutils_attributes.Rd diff --git a/man/check_duplicates.Rd b/man/check_duplicates.Rd index dff552eb5..cb71be3b1 100644 --- a/man/check_duplicates.Rd +++ b/man/check_duplicates.Rd @@ -10,8 +10,8 @@ check_duplicates(data, forecast_unit = NULL) \item{data}{A data.frame as used for \code{\link[=score]{score()}}} \item{forecast_unit}{A character vector with the column names that define -the unit of a single forecast. If \code{NULL} (the default) the function tries -to infer the unit of a single forecast.} +the unit of a single forecast. By default the forecast unit will be +automatically inferred from the data (see \code{\link[=get_forecast_unit]{get_forecast_unit()}})} } \value{ Returns TRUE if the check was successful and a string with an diff --git a/man/document_score_data.Rd b/man/document_score_data.Rd deleted file mode 100644 index 9b8b19558..000000000 --- a/man/document_score_data.Rd +++ /dev/null @@ -1,15 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/documentation-templates.R -\name{document_score_data} -\alias{document_score_data} -\title{Documentation template for scoring input data} -\arguments{ -\item{data}{A data frame (or similar) of forecasts following the -specifications detailed in \code{\link[=score]{score()}}.} - -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} -} -\description{ -Documentation template for scoring input data -} -\keyword{internal} diff --git a/man/get_duplicate_forecasts.Rd b/man/get_duplicate_forecasts.Rd index 26f77e857..2d39cc05a 100644 --- a/man/get_duplicate_forecasts.Rd +++ b/man/get_duplicate_forecasts.Rd @@ -2,16 +2,16 @@ % Please edit documentation in R/get_-functions.R \name{get_duplicate_forecasts} \alias{get_duplicate_forecasts} -\title{Find duplicate forecasts} +\title{Find Duplicate Forecasts} \usage{ -get_duplicate_forecasts(data, forecast_unit = NULL) +get_duplicate_forecasts(data, forecast_unit = get_forecast_unit(data)) } \arguments{ \item{data}{A data.frame as used for \code{\link[=score]{score()}}} \item{forecast_unit}{A character vector with the column names that define -the unit of a single forecast. If \code{NULL} (the default) the function tries -to infer the unit of a single forecast.} +the unit of a single forecast. By default the forecast unit will be +automatically inferred from the data (see \code{\link[=get_forecast_unit]{get_forecast_unit()}})} } \value{ A data.frame with all rows for which a duplicate forecast was found diff --git a/man/get_forecast_unit.Rd b/man/get_forecast_unit.Rd index ff2785030..0a0e659b2 100644 --- a/man/get_forecast_unit.Rd +++ b/man/get_forecast_unit.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/get_-functions.R \name{get_forecast_unit} \alias{get_forecast_unit} -\title{Get unit of a single forecast} +\title{Get Unit Of A Single Forecast} \usage{ get_forecast_unit(data) } diff --git a/man/get_protected_columns.Rd b/man/get_protected_columns.Rd index 4bbf8f6da..2d759ba52 100644 --- a/man/get_protected_columns.Rd +++ b/man/get_protected_columns.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/get_-functions.R \name{get_protected_columns} \alias{get_protected_columns} -\title{Get protected columns from a data frame} +\title{Get Protected Columns From Data} \usage{ get_protected_columns(data = NULL) } diff --git a/man/get_scoringutils_attributes.Rd b/man/get_scoringutils_attributes.Rd deleted file mode 100644 index 5f2770a6a..000000000 --- a/man/get_scoringutils_attributes.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/get_-functions.R -\name{get_scoringutils_attributes} -\alias{get_scoringutils_attributes} -\title{Get a list of all attributes of a scoringutils object} -\usage{ -get_scoringutils_attributes(object) -} -\arguments{ -\item{object}{A object of class \code{forecast_}} -} -\value{ -A named list with the attributes of that object. -} -\description{ -Get a list of all attributes of a scoringutils object -} -\keyword{internal} diff --git a/man/transform_forecasts.Rd b/man/transform_forecasts.Rd index a77043a95..4c7a5f1e9 100644 --- a/man/transform_forecasts.Rd +++ b/man/transform_forecasts.Rd @@ -33,11 +33,11 @@ newly transformed values. Only relevant if \code{append = TRUE}.} the default option of \code{\link[=log_shift]{log_shift()}} this could be the \code{offset} argument.} } \value{ -A \code{data.table} with either a transformed version of the data, or one -with both the untransformed and the transformed data. includes the original -data as well as a transformation of the original data. There will be one -additional column, `scale', present which will be set to "natural" for the -untransformed forecasts. +A forecast object with either a transformed version of the data, or +one with both the untransformed and the transformed data. includes the +original data as well as a transformation of the original data. There will +be one additional column, `scale', present which will be set to "natural" +for the untransformed forecasts. } \description{ Function to transform forecasts and observed values before From 555d76b512f1fcd69190eee5c22af1c870ce48bc Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 6 Mar 2024 23:56:55 -0600 Subject: [PATCH 10/22] Change some docs to title case --- R/metrics-sample.R | 10 +++++----- man/ae_median_sample.Rd | 2 +- man/bias_sample.Rd | 2 +- man/logs_sample.Rd | 2 +- man/mad_sample.Rd | 4 ++-- man/se_mean_sample.Rd | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/R/metrics-sample.R b/R/metrics-sample.R index 044ca8ee0..7cc2c5ec7 100644 --- a/R/metrics-sample.R +++ b/R/metrics-sample.R @@ -1,4 +1,4 @@ -#' @title Determines bias of forecasts +#' @title Determines Bias Of Forecasts #' #' @description #' Determines bias from predictive Monte-Carlo samples. The function @@ -76,7 +76,7 @@ bias_sample <- function(observed, predicted) { } -#' @title Absolute Error of the Median (Sample-based Version) +#' @title Absolute Error Of The Median (Sample-Based Version) #' #' @description #' Absolute error of the median calculated as @@ -111,7 +111,7 @@ ae_median_sample <- function(observed, predicted) { } -#' @title Squared Error of the Mean (Sample-based Version) +#' @title Squared Error Of The Mean (Sample-Based Version) #' #' @description #' Squared error of the mean calculated as @@ -143,7 +143,7 @@ se_mean_sample <- function(observed, predicted) { } -#' @title Logarithmic score +#' @title Logarithmic Score #' #' @description #' Wrapper around the [`logs_sample()`][scoringRules::scores_sample_univ] @@ -243,7 +243,7 @@ crps_sample <- function(observed, predicted, ...) { } -#' @title Determine dispersion of a probabilistic forecast +#' @title Determine Dispersion Of A Probabilistic Forecast #' @details #' Sharpness is the ability of the model to generate predictions within a #' narrow range and dispersion is the lack thereof. diff --git a/man/ae_median_sample.Rd b/man/ae_median_sample.Rd index 6967f44fd..5da8b1aa9 100644 --- a/man/ae_median_sample.Rd +++ b/man/ae_median_sample.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/metrics-sample.R \name{ae_median_sample} \alias{ae_median_sample} -\title{Absolute Error of the Median (Sample-based Version)} +\title{Absolute Error Of The Median (Sample-Based Version)} \usage{ ae_median_sample(observed, predicted) } diff --git a/man/bias_sample.Rd b/man/bias_sample.Rd index 98d7e19d5..6fbfe4496 100644 --- a/man/bias_sample.Rd +++ b/man/bias_sample.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/metrics-sample.R \name{bias_sample} \alias{bias_sample} -\title{Determines bias of forecasts} +\title{Determines Bias Of Forecasts} \usage{ bias_sample(observed, predicted) } diff --git a/man/logs_sample.Rd b/man/logs_sample.Rd index aebb6ac5c..65649acf3 100644 --- a/man/logs_sample.Rd +++ b/man/logs_sample.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/metrics-sample.R \name{logs_sample} \alias{logs_sample} -\title{Logarithmic score} +\title{Logarithmic Score} \usage{ logs_sample(observed, predicted, ...) } diff --git a/man/mad_sample.Rd b/man/mad_sample.Rd index 7594caa8e..1cd75b170 100644 --- a/man/mad_sample.Rd +++ b/man/mad_sample.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/metrics-sample.R \name{mad_sample} \alias{mad_sample} -\title{Determine dispersion of a probabilistic forecast} +\title{Determine Dispersion Of A Probabilistic Forecast} \usage{ mad_sample(observed = NULL, predicted, ...) } @@ -21,7 +21,7 @@ Carlo samples. Alternatively, \code{predicted} can just be a vector of size n.} vector with dispersion values } \description{ -Determine dispersion of a probabilistic forecast +Determine Dispersion Of A Probabilistic Forecast } \details{ Sharpness is the ability of the model to generate predictions within a diff --git a/man/se_mean_sample.Rd b/man/se_mean_sample.Rd index 4a5d62aab..c5cced3a5 100644 --- a/man/se_mean_sample.Rd +++ b/man/se_mean_sample.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/metrics-sample.R \name{se_mean_sample} \alias{se_mean_sample} -\title{Squared Error of the Mean (Sample-based Version)} +\title{Squared Error Of The Mean (Sample-Based Version)} \usage{ se_mean_sample(observed, predicted) } From 9d2afa7eade6e93ccfd05c4ece5d6c95359570c0 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:03:45 -0600 Subject: [PATCH 11/22] Update docs for pairwise comparison functions --- R/pairwise-comparisons.R | 6 ++++-- man/compare_two_models.Rd | 4 ++++ man/pairwise_comparison.Rd | 2 +- man/pairwise_comparison_one_group.Rd | 3 +++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/R/pairwise-comparisons.R b/R/pairwise-comparisons.R index 18f3ed0b8..90e09cfc6 100644 --- a/R/pairwise-comparisons.R +++ b/R/pairwise-comparisons.R @@ -41,7 +41,7 @@ #' model against which to compare other models. #' @param ... additional arguments for the comparison between two models. See #' [compare_two_models()] for more information. -#' @return A ggplot2 object with a coloured table of summarised scores +#' @return A data.table with pairwise comparisons #' @importFrom data.table as.data.table data.table setnames copy #' @importFrom stats sd rbinom wilcox.test p.adjust #' @importFrom utils combn @@ -179,7 +179,7 @@ pairwise_comparison <- function( #' subgroup is managed from [pairwise_comparison_one_group()]. In order to #' actually do the comparison between two models over a subset of common #' forecasts it calls [compare_two_models()]. -#' @inheritParams pairwise_comparison +#' @inherit pairwise_comparison params return #' @keywords internal pairwise_comparison_one_group <- function(scores, @@ -322,6 +322,8 @@ pairwise_comparison_one_group <- function(scores, #' determine p-values. #' @param n_permutations numeric, the number of permutations for a #' permutation test. Default is 999. +#' @return A list with mean score ratios and p-values for the comparison +#' between two models #' @author Johannes Bracher, \email{johannes.bracher@@kit.edu} #' @author Nikos Bosse \email{nikosbosse@@gmail.com} #' @keywords internal diff --git a/man/compare_two_models.Rd b/man/compare_two_models.Rd index cfe4fafa6..4ce44e1fd 100644 --- a/man/compare_two_models.Rd +++ b/man/compare_two_models.Rd @@ -35,6 +35,10 @@ determine p-values.} \item{n_permutations}{numeric, the number of permutations for a permutation test. Default is 999.} } +\value{ +A list with mean score ratios and p-values for the comparison +between two models +} \description{ This function compares two models based on the subset of forecasts for which both models have made a prediction. It gets called diff --git a/man/pairwise_comparison.Rd b/man/pairwise_comparison.Rd index 499a1bba6..d9f00692a 100644 --- a/man/pairwise_comparison.Rd +++ b/man/pairwise_comparison.Rd @@ -34,7 +34,7 @@ model against which to compare other models.} \code{\link[=compare_two_models]{compare_two_models()}} for more information.} } \value{ -A ggplot2 object with a coloured table of summarised scores +A data.table with pairwise comparisons } \description{ Compute relative scores between different models making pairwise diff --git a/man/pairwise_comparison_one_group.Rd b/man/pairwise_comparison_one_group.Rd index d52211559..b98fa1b55 100644 --- a/man/pairwise_comparison_one_group.Rd +++ b/man/pairwise_comparison_one_group.Rd @@ -27,6 +27,9 @@ split data.frames.} \item{...}{additional arguments for the comparison between two models. See \code{\link[=compare_two_models]{compare_two_models()}} for more information.} } +\value{ +A data.table with pairwise comparisons +} \description{ This function does the pairwise comparison for one set of forecasts, but multiple models involved. It gets called from \code{\link[=pairwise_comparison]{pairwise_comparison()}}. From 6118a2563e021928ea74f4c22f15efc8c45e2aa5 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:11:10 -0600 Subject: [PATCH 12/22] Update return statements for plots --- R/plot.R | 16 ++++++++++------ man/plot_correlation.Rd | 4 +++- man/plot_forecast_counts.Rd | 2 +- man/plot_heatmap.Rd | 2 +- man/plot_pairwise_comparison.Rd | 4 ++++ man/plot_pit.Rd | 2 +- man/plot_score_table.Rd | 2 +- man/plot_wis.Rd | 4 +++- 8 files changed, 24 insertions(+), 12 deletions(-) diff --git a/R/plot.R b/R/plot.R index c051696b9..dc4f11fbb 100644 --- a/R/plot.R +++ b/R/plot.R @@ -14,7 +14,7 @@ #' @param metrics A character vector with the metrics to show. If set to #' `NULL` (default), all metrics present in `scores` will be shown. #' -#' @return A ggplot2 object with a coloured table of summarised scores +#' @return A ggplot object with a coloured table of summarised scores #' @inheritParams pairwise_comparison #' @importFrom ggplot2 ggplot aes element_blank element_text labs coord_cartesian coord_flip #' @importFrom data.table setDT melt @@ -140,11 +140,12 @@ plot_score_table <- function(scores, #' @param relative_contributions show relative contributions instead of absolute #' contributions. Default is FALSE and this functionality is not available yet. #' @param flip boolean (default is `FALSE`), whether or not to flip the axes. -#' @return A ggplot2 object showing a contributions from the three components of +#' @return A ggplot object showing a contributions from the three components of #' the weighted interval score #' @importFrom ggplot2 ggplot aes geom_linerange facet_wrap labs #' scale_fill_discrete #' theme theme_light unit guides guide_legend .data +#' @return A ggplot object with a visualisation of the WIS decomposition #' @export #' @examples #' library(ggplot2) @@ -226,7 +227,7 @@ plot_wis <- function(scores, #' could be something like "horizon", or "location" #' @param metric the metric that determines the value and colour shown in the #' tiles of the heatmap -#' @return A ggplot2 object showing a heatmap of the desired metric +#' @return A ggplot object showing a heatmap of the desired metric #' @importFrom data.table setDT `:=` #' @importFrom ggplot2 ggplot aes geom_tile geom_text .data #' scale_fill_gradient2 labs element_text coord_cartesian @@ -410,6 +411,8 @@ plot_quantile_coverage <- function(coverage, #' @importFrom stats reorder #' @importFrom ggplot2 labs coord_cartesian facet_wrap facet_grid theme #' element_text element_blank +#' @return A ggplot object with a heatmap of mean score ratios from pairwise +#' comparisons #' @export #' @examples #' library(ggplot2) @@ -541,7 +544,7 @@ plot_pairwise_comparison <- function(comparison_result, #' @importFrom stats as.formula #' @importFrom ggplot2 geom_col #' @importFrom stats density -#' @return vector with the scoring values +#' @return A ggplot object with a histogram of PIT values #' @examples #' \dontshow{ #' data.table::setDTthreads(2) # restricts number of cores used on CRAN @@ -662,7 +665,7 @@ plot_pit <- function(pit, #' are shown on the x-axis. #' @param show_counts logical (default is `TRUE`) that indicates whether #' or not to show the actual count numbers on the plot -#' @return ggplot object with a plot of interval coverage +#' @return ggplot object with a plot of forecast counts #' @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual #' geom_tile scale_fill_gradient .data #' @importFrom data.table dcast .I .N @@ -730,12 +733,13 @@ plot_forecast_counts <- function(forecast_counts, #' #' @param correlations A data.table of correlations between scores as produced #' by [correlation()]. -#' @return A ggplot2 object showing a coloured matrix of correlations +#' @return A ggplot object showing a coloured matrix of correlations #' between metrics #' @importFrom ggplot2 ggplot geom_tile geom_text aes scale_fill_gradient2 #' element_text labs coord_cartesian theme element_blank #' @importFrom data.table setDT melt #' @export +#' @return A ggplot object with a visualisation of correlations between metrics #' @examples #' scores <- score(example_quantile) #' correlations <- correlation( diff --git a/man/plot_correlation.Rd b/man/plot_correlation.Rd index 0bade90cb..97cbf4056 100644 --- a/man/plot_correlation.Rd +++ b/man/plot_correlation.Rd @@ -11,8 +11,10 @@ plot_correlation(correlations) by \code{\link[=correlation]{correlation()}}.} } \value{ -A ggplot2 object showing a coloured matrix of correlations +A ggplot object showing a coloured matrix of correlations between metrics + +A ggplot object with a visualisation of correlations between metrics } \description{ Plots a heatmap of correlations between different metrics diff --git a/man/plot_forecast_counts.Rd b/man/plot_forecast_counts.Rd index c5285c9dd..44a57a7fe 100644 --- a/man/plot_forecast_counts.Rd +++ b/man/plot_forecast_counts.Rd @@ -30,7 +30,7 @@ are shown on the x-axis.} or not to show the actual count numbers on the plot} } \value{ -ggplot object with a plot of interval coverage +ggplot object with a plot of forecast counts } \description{ Visualise Where Forecasts Are Available diff --git a/man/plot_heatmap.Rd b/man/plot_heatmap.Rd index 837ef4243..8e47af4c6 100644 --- a/man/plot_heatmap.Rd +++ b/man/plot_heatmap.Rd @@ -20,7 +20,7 @@ could be something like "horizon", or "location"} tiles of the heatmap} } \value{ -A ggplot2 object showing a heatmap of the desired metric +A ggplot object showing a heatmap of the desired metric } \description{ This function can be used to create a heatmap of one metric across different diff --git a/man/plot_pairwise_comparison.Rd b/man/plot_pairwise_comparison.Rd index 71b5e77e8..64054a307 100644 --- a/man/plot_pairwise_comparison.Rd +++ b/man/plot_pairwise_comparison.Rd @@ -18,6 +18,10 @@ plot_pairwise_comparison( visualise the ratio or the p-value of the pairwise comparison. Default is "mean_scores_ratio".} } +\value{ +A ggplot object with a heatmap of mean score ratios from pairwise +comparisons +} \description{ Creates a heatmap of the ratios or pvalues from a pairwise comparison between models diff --git a/man/plot_pit.Rd b/man/plot_pit.Rd index 7aafebed2..149509ef5 100644 --- a/man/plot_pit.Rd +++ b/man/plot_pit.Rd @@ -25,7 +25,7 @@ quantile-based data. Default is \code{NULL} and breaks will be determined by \code{num_bins}.} } \value{ -vector with the scoring values +A ggplot object with a histogram of PIT values } \description{ Make a simple histogram of the probability integral transformed values to diff --git a/man/plot_score_table.Rd b/man/plot_score_table.Rd index 1c1962272..88257272f 100644 --- a/man/plot_score_table.Rd +++ b/man/plot_score_table.Rd @@ -22,7 +22,7 @@ metric, but you can provide additional column names (see examples).} \code{NULL} (default), all metrics present in \code{scores} will be shown.} } \value{ -A ggplot2 object with a coloured table of summarised scores +A ggplot object with a coloured table of summarised scores } \description{ Plots a coloured table of summarised scores obtained using diff --git a/man/plot_wis.Rd b/man/plot_wis.Rd index 982ff1406..8f2418c18 100644 --- a/man/plot_wis.Rd +++ b/man/plot_wis.Rd @@ -19,8 +19,10 @@ contributions. Default is FALSE and this functionality is not available yet.} \item{flip}{boolean (default is \code{FALSE}), whether or not to flip the axes.} } \value{ -A ggplot2 object showing a contributions from the three components of +A ggplot object showing a contributions from the three components of the weighted interval score + +A ggplot object with a visualisation of the WIS decomposition } \description{ Visualise the components of the weighted interval score: penalties for From 7b174a4b8e9ecb4f3e887651dbeb3658fc9b6ecc Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:16:48 -0600 Subject: [PATCH 13/22] Update return stament for `score()` and related functions --- R/score.R | 15 +++++---------- man/as_scores.Rd | 2 +- man/new_scores.Rd | 3 +++ man/score.Rd | 11 +++-------- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/R/score.R b/R/score.R index b65bcd835..c30ca0715 100644 --- a/R/score.R +++ b/R/score.R @@ -20,14 +20,9 @@ #' [rules_quantile()], and [rules_sample()] for more information on the #' default metrics used. #' @param ... additional arguments -#' @return A data.table with unsummarised scores. This will generally be -#' one score per forecast (as defined by the unit of a single forecast). -#' -#' For quantile-based forecasts, one score per quantile will be returned -#' instead. This is done as scores can be computed and may be of interest -#' for individual quantiles. You can call [summarise_scores()]) on the -#' unsummarised scores to obtain one score per forecast unit for quantile-based -#' forecasts. +#' @return A data.table with unsummarised scores, i.e. with one score per +#' forecast. See [summarise_scores()]) for information on how to summarise +#' scores. #' @importFrom data.table ':=' as.data.table #' @importFrom stats na.omit #' @examples @@ -230,6 +225,7 @@ apply_rules <- function(data, metrics, ...) { #' @param score_names A character vector with the names of the scores #' (i.e. the names of the scoring rules used for scoring) #' @keywords internal +#' @return An object of class `scores` #' @examples #' \dontrun{ #' df <- data.frame( @@ -249,8 +245,7 @@ new_scores <- function(scores, score_names) { #' Create An Object Of Class `scores` From Data #' @description This convenience function wraps [new_scores()] and validates #' the `scores` object. -#' @inheritParams new_scores -#' @returns Returns an object of class 1scores` +#' @inherit new_scores params return #' @importFrom checkmate assert_data_frame #' @keywords internal as_scores <- function(scores, score_names) { diff --git a/man/as_scores.Rd b/man/as_scores.Rd index 2f636ebff..ce71327bf 100644 --- a/man/as_scores.Rd +++ b/man/as_scores.Rd @@ -13,7 +13,7 @@ as_scores(scores, score_names) (i.e. the names of the scoring rules used for scoring)} } \value{ -Returns an object of class 1scores` +An object of class \code{scores} } \description{ This convenience function wraps \code{\link[=new_scores]{new_scores()}} and validates diff --git a/man/new_scores.Rd b/man/new_scores.Rd index 083e0409d..a10175196 100644 --- a/man/new_scores.Rd +++ b/man/new_scores.Rd @@ -12,6 +12,9 @@ new_scores(scores, score_names) \item{score_names}{A character vector with the names of the scores (i.e. the names of the scoring rules used for scoring)} } +\value{ +An object of class \code{scores} +} \description{ This function creates an object of class \code{scores} based on a data.table or similar. diff --git a/man/score.Rd b/man/score.Rd index 88e3d2b9d..a7f7fc850 100644 --- a/man/score.Rd +++ b/man/score.Rd @@ -32,14 +32,9 @@ column names in the output. See \code{\link[=rules_point]{rules_point()}}, \code default metrics used.} } \value{ -A data.table with unsummarised scores. This will generally be -one score per forecast (as defined by the unit of a single forecast). - -For quantile-based forecasts, one score per quantile will be returned -instead. This is done as scores can be computed and may be of interest -for individual quantiles. You can call \code{\link[=summarise_scores]{summarise_scores()}}) on the -unsummarised scores to obtain one score per forecast unit for quantile-based -forecasts. +A data.table with unsummarised scores, i.e. with one score per +forecast. See \code{\link[=summarise_scores]{summarise_scores()}}) for information on how to summarise +scores. } \description{ \code{score()} applies a selection of scoring metrics to a data.frame From 76342445146d01d0c905e95dc04361f00dd9d9d0 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:33:08 -0600 Subject: [PATCH 14/22] Update docs for `pairwise_comparison()` and `add_pairwise_comparison()` and related functions --- R/pairwise-comparisons.R | 27 +++++++++++++----------- R/summarise_scores.R | 31 ++++++---------------------- man/add_pairwise_comparison.Rd | 25 +++++++++++++--------- man/compare_two_models.Rd | 8 ++++--- man/correlation.Rd | 3 ++- man/infer_rel_skill_metric.Rd | 3 ++- man/pairwise_comparison.Rd | 28 ++++++++++++++----------- man/pairwise_comparison_one_group.Rd | 28 ++++++++++++++----------- man/plot_score_table.Rd | 3 ++- man/summarise_scores.Rd | 15 +++++--------- 10 files changed, 84 insertions(+), 87 deletions(-) diff --git a/R/pairwise-comparisons.R b/R/pairwise-comparisons.R index 90e09cfc6..be2e66c87 100644 --- a/R/pairwise-comparisons.R +++ b/R/pairwise-comparisons.R @@ -26,21 +26,24 @@ #' `permutationTest` from the `surveillance` package by Michael Höhle, #' Andrea Riebler and Michaela Paul. #' -#' @param scores A data.table of scores as produced by [score()]. -#' @param metric A character vector of length one with the metric to do the -#' comparison on. -#' @param by character vector with names of columns present in the input -#' data.frame. `by` determines how pairwise comparisons will be computed. -#' You will get a relative skill score for every grouping level determined in -#' `by`. If, for example, `by = c("model", "location")`. Then you will get a +#' @param by character vector with column names that define the grouping level +#' for the pairwise comparisons. By default (`model`), there will be one +#' relative skill score per model. If, for example, +#' `by = c("model", "location")`. Then you will get a #' separate relative skill score for every model in every location. Internally, -#' the data.frame will be split according `by` (but removing "model" before -#' splitting) and the pairwise comparisons will be computed separately for the -#' split data.frames. -#' @param baseline character vector of length one that denotes the baseline -#' model against which to compare other models. +#' the data.table with scores will be split according `by` (removing "model" +#' before splitting) and the pairwise comparisons will be computed separately +#' for the split data.tables. +#' @param metric A string with the name of the metric for which +#' a relative skill shall be computed. By default this is either "crps", +#' "wis" or "brier_score" if any of these are available. +#' @param baseline A string with the name of a model. If a baseline is +#' given, then a scaled relative skill with respect to the baseline will be +#' returned. By default (`NULL`), relative skill will not be scaled with +#' respect to a baseline model. #' @param ... additional arguments for the comparison between two models. See #' [compare_two_models()] for more information. +#' @inheritParams summarise_scores #' @return A data.table with pairwise comparisons #' @importFrom data.table as.data.table data.table setnames copy #' @importFrom stats sd rbinom wilcox.test p.adjust diff --git a/R/summarise_scores.R b/R/summarise_scores.R index 02e73dcdf..bce17519b 100644 --- a/R/summarise_scores.R +++ b/R/summarise_scores.R @@ -10,21 +10,15 @@ #' do, you also have to manually update the attribute by calling #' `attr(scores, "score_names") <- new_names`. #' -#' @inheritParams pairwise_comparison -#' @inheritParams score +#' @param scores An object of class `scores` (a data.table with +#' scores and an additional attribute `score_names` as produced by [score()]) #' @param by character vector with column names to summarise scores by. Default #' is `model`, meaning that there will be one score per model in the output. -#' The *unit of a single forecast* is determined by the columns present in the -#' input data that do not correspond to a metric produced by [score()], which -#' indicate indicate a grouping of forecasts (for example there may be one -#' forecast per day, location and model). Adding additional, unrelated, columns -#' may alter results in an unpredictable way. -#' @param across character vector with column names from the vector of variables -#' that define the *unit of a single forecast* (see above) to summarise scores +#' @param across character vector with column names to summarise scores #' across (meaning that the specified columns will be dropped). This is an #' alternative to specifying `by` directly. If `across` is set, `by` will be #' ignored. If `across` is `NULL` (default), then `by` will be used. -#' @param fun a function used for summarising scores. Default is `mean`. +#' @param fun a function used for summarising scores. Default is [mean()]. #' @param ... additional parameters that can be passed to the summary function #' provided to `fun`. For more information see the documentation of the #' respective function. @@ -115,24 +109,11 @@ summarize_scores <- summarise_scores #' @title Add pairwise comparisons #' @description Adds a columns with relative skills computed by running #' pairwise comparisons on the scores. -#' -#' a column called -#' 'model' must be present in the input data. For more information on +#' For more information on #' the computation of relative skill, see [pairwise_comparison()]. #' Relative skill will be calculated for the aggregation level specified in #' `by`. -#' WRITE MORE INFO HERE. -#' -#' -#' @param scores MORE INFO HERE. -#' @param by character vector with column names to summarise scores by. Default -#' is "model", meaning that there will be one relative skill score per model. -#' @param metric character with the name of the metric for which -#' a relative skill shall be computed. -#' @param baseline character string with the name of a model. If a baseline is -#' given, then a scaled relative skill with respect to the baseline will be -#' returned. By default (`NULL`), relative skill will not be scaled with -#' respect to a baseline model. +#' @inheritParams pairwise_comparison #' @export #' @keywords keyword scoring add_pairwise_comparison <- function( diff --git a/man/add_pairwise_comparison.Rd b/man/add_pairwise_comparison.Rd index cd16e14a0..73f49053d 100644 --- a/man/add_pairwise_comparison.Rd +++ b/man/add_pairwise_comparison.Rd @@ -12,15 +12,23 @@ add_pairwise_comparison( ) } \arguments{ -\item{scores}{MORE INFO HERE.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} -\item{by}{character vector with column names to summarise scores by. Default -is "model", meaning that there will be one relative skill score per model.} +\item{by}{character vector with column names that define the grouping level +for the pairwise comparisons. By default (\code{model}), there will be one +relative skill score per model. If, for example, +\code{by = c("model", "location")}. Then you will get a +separate relative skill score for every model in every location. Internally, +the data.table with scores will be split according \code{by} (removing "model" +before splitting) and the pairwise comparisons will be computed separately +for the split data.tables.} -\item{metric}{character with the name of the metric for which -a relative skill shall be computed.} +\item{metric}{A string with the name of the metric for which +a relative skill shall be computed. By default this is either "crps", +"wis" or "brier_score" if any of these are available.} -\item{baseline}{character string with the name of a model. If a baseline is +\item{baseline}{A string with the name of a model. If a baseline is given, then a scaled relative skill with respect to the baseline will be returned. By default (\code{NULL}), relative skill will not be scaled with respect to a baseline model.} @@ -28,13 +36,10 @@ respect to a baseline model.} \description{ Adds a columns with relative skills computed by running pairwise comparisons on the scores. - -a column called -'model' must be present in the input data. For more information on +For more information on the computation of relative skill, see \code{\link[=pairwise_comparison]{pairwise_comparison()}}. Relative skill will be calculated for the aggregation level specified in \code{by}. -WRITE MORE INFO HERE. } \keyword{keyword} \keyword{scoring} diff --git a/man/compare_two_models.Rd b/man/compare_two_models.Rd index 4ce44e1fd..752c8c608 100644 --- a/man/compare_two_models.Rd +++ b/man/compare_two_models.Rd @@ -15,14 +15,16 @@ compare_two_models( ) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} \item{name_model1}{character, name of the first model} \item{name_model2}{character, name of the model to compare against} -\item{metric}{A character vector of length one with the metric to do the -comparison on.} +\item{metric}{A string with the name of the metric for which +a relative skill shall be computed. By default this is either "crps", +"wis" or "brier_score" if any of these are available.} \item{one_sided}{Boolean, default is \code{FALSE}, whether two conduct a one-sided instead of a two-sided test to determine significance in a pairwise diff --git a/man/correlation.Rd b/man/correlation.Rd index 59bda9e42..f804187a7 100644 --- a/man/correlation.Rd +++ b/man/correlation.Rd @@ -7,7 +7,8 @@ correlation(scores, metrics = NULL, digits = NULL) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} \item{metrics}{A character vector with the metrics to show. If set to \code{NULL} (default), all metrics present in \code{scores} will diff --git a/man/infer_rel_skill_metric.Rd b/man/infer_rel_skill_metric.Rd index 323cd84fd..c0914820b 100644 --- a/man/infer_rel_skill_metric.Rd +++ b/man/infer_rel_skill_metric.Rd @@ -7,7 +7,8 @@ infer_rel_skill_metric(scores) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} } \description{ Helper function to infer the metric for which pairwise comparisons shall diff --git a/man/pairwise_comparison.Rd b/man/pairwise_comparison.Rd index d9f00692a..419a2eefb 100644 --- a/man/pairwise_comparison.Rd +++ b/man/pairwise_comparison.Rd @@ -13,22 +13,26 @@ pairwise_comparison( ) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} -\item{by}{character vector with names of columns present in the input -data.frame. \code{by} determines how pairwise comparisons will be computed. -You will get a relative skill score for every grouping level determined in -\code{by}. If, for example, \code{by = c("model", "location")}. Then you will get a +\item{by}{character vector with column names that define the grouping level +for the pairwise comparisons. By default (\code{model}), there will be one +relative skill score per model. If, for example, +\code{by = c("model", "location")}. Then you will get a separate relative skill score for every model in every location. Internally, -the data.frame will be split according \code{by} (but removing "model" before -splitting) and the pairwise comparisons will be computed separately for the -split data.frames.} +the data.table with scores will be split according \code{by} (removing "model" +before splitting) and the pairwise comparisons will be computed separately +for the split data.tables.} -\item{metric}{A character vector of length one with the metric to do the -comparison on.} +\item{metric}{A string with the name of the metric for which +a relative skill shall be computed. By default this is either "crps", +"wis" or "brier_score" if any of these are available.} -\item{baseline}{character vector of length one that denotes the baseline -model against which to compare other models.} +\item{baseline}{A string with the name of a model. If a baseline is +given, then a scaled relative skill with respect to the baseline will be +returned. By default (\code{NULL}), relative skill will not be scaled with +respect to a baseline model.} \item{...}{additional arguments for the comparison between two models. See \code{\link[=compare_two_models]{compare_two_models()}} for more information.} diff --git a/man/pairwise_comparison_one_group.Rd b/man/pairwise_comparison_one_group.Rd index b98fa1b55..90ef41e8f 100644 --- a/man/pairwise_comparison_one_group.Rd +++ b/man/pairwise_comparison_one_group.Rd @@ -7,22 +7,26 @@ pairwise_comparison_one_group(scores, metric, baseline, by, ...) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} -\item{metric}{A character vector of length one with the metric to do the -comparison on.} +\item{metric}{A string with the name of the metric for which +a relative skill shall be computed. By default this is either "crps", +"wis" or "brier_score" if any of these are available.} -\item{baseline}{character vector of length one that denotes the baseline -model against which to compare other models.} +\item{baseline}{A string with the name of a model. If a baseline is +given, then a scaled relative skill with respect to the baseline will be +returned. By default (\code{NULL}), relative skill will not be scaled with +respect to a baseline model.} -\item{by}{character vector with names of columns present in the input -data.frame. \code{by} determines how pairwise comparisons will be computed. -You will get a relative skill score for every grouping level determined in -\code{by}. If, for example, \code{by = c("model", "location")}. Then you will get a +\item{by}{character vector with column names that define the grouping level +for the pairwise comparisons. By default (\code{model}), there will be one +relative skill score per model. If, for example, +\code{by = c("model", "location")}. Then you will get a separate relative skill score for every model in every location. Internally, -the data.frame will be split according \code{by} (but removing "model" before -splitting) and the pairwise comparisons will be computed separately for the -split data.frames.} +the data.table with scores will be split according \code{by} (removing "model" +before splitting) and the pairwise comparisons will be computed separately +for the split data.tables.} \item{...}{additional arguments for the comparison between two models. See \code{\link[=compare_two_models]{compare_two_models()}} for more information.} diff --git a/man/plot_score_table.Rd b/man/plot_score_table.Rd index 88257272f..0af9065b8 100644 --- a/man/plot_score_table.Rd +++ b/man/plot_score_table.Rd @@ -7,7 +7,8 @@ plot_score_table(scores, y = "model", by = NULL, metrics = NULL) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} \item{y}{the variable to be shown on the y-axis. Instead of a single character string, you can also specify a vector with column names, e.g. diff --git a/man/summarise_scores.Rd b/man/summarise_scores.Rd index d18ca4e38..94860d199 100644 --- a/man/summarise_scores.Rd +++ b/man/summarise_scores.Rd @@ -10,23 +10,18 @@ summarise_scores(scores, by = "model", across = NULL, fun = mean, ...) summarize_scores(scores, by = "model", across = NULL, fun = mean, ...) } \arguments{ -\item{scores}{A data.table of scores as produced by \code{\link[=score]{score()}}.} +\item{scores}{An object of class \code{scores} (a data.table with +scores and an additional attribute \code{score_names} as produced by \code{\link[=score]{score()}})} \item{by}{character vector with column names to summarise scores by. Default -is \code{model}, meaning that there will be one score per model in the output. -The \emph{unit of a single forecast} is determined by the columns present in the -input data that do not correspond to a metric produced by \code{\link[=score]{score()}}, which -indicate indicate a grouping of forecasts (for example there may be one -forecast per day, location and model). Adding additional, unrelated, columns -may alter results in an unpredictable way.} +is \code{model}, meaning that there will be one score per model in the output.} -\item{across}{character vector with column names from the vector of variables -that define the \emph{unit of a single forecast} (see above) to summarise scores +\item{across}{character vector with column names to summarise scores across (meaning that the specified columns will be dropped). This is an alternative to specifying \code{by} directly. If \code{across} is set, \code{by} will be ignored. If \code{across} is \code{NULL} (default), then \code{by} will be used.} -\item{fun}{a function used for summarising scores. Default is \code{mean}.} +\item{fun}{a function used for summarising scores. Default is \code{\link[=mean]{mean()}}.} \item{...}{additional parameters that can be passed to the summary function provided to \code{fun}. For more information see the documentation of the From e31fe1a50277717a7b5cdf7ba6af3583de92e17e Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:36:07 -0600 Subject: [PATCH 15/22] remove input check from get_forecast_unit() due to failing checks --- R/get_-functions.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/R/get_-functions.R b/R/get_-functions.R index a6390cea1..5c81cf117 100644 --- a/R/get_-functions.R +++ b/R/get_-functions.R @@ -180,11 +180,9 @@ get_score_names <- function(scores, error = FALSE) { #' @inheritParams validate_forecast #' @return A character vector with the column names that define the unit of #' a single forecast -#' @importFrom checkmate assert_data_frame #' @export #' @keywords check-forecasts get_forecast_unit <- function(data) { - assert_data_frame(data) protected_columns <- get_protected_columns(data) protected_columns <- c(protected_columns, attr(data, "score_names")) forecast_unit <- setdiff(colnames(data), unique(protected_columns)) From 429149ca41eb5e7b69a86af27bb8019399aabae1 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:36:17 -0600 Subject: [PATCH 16/22] Update test --- tests/testthat/test-get_-functions.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-get_-functions.R b/tests/testthat/test-get_-functions.R index f40307b7e..da80d7cef 100644 --- a/tests/testthat/test-get_-functions.R +++ b/tests/testthat/test-get_-functions.R @@ -35,9 +35,9 @@ test_that("removing NA rows from data works as expected", { c("forecast_sample", "data.table", "data.frame") ) - attributes <- get_scoringutils_attributes(ex) + attributes <- attributes(ex) expect_equal( - get_scoringutils_attributes(na.omit(ex)), + attributes(na.omit(ex)), attributes ) }) From 9e4f832400dfa914f9c018dd015ba881ca3296df Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:39:19 -0600 Subject: [PATCH 17/22] Update docs for data-wrangling functions --- R/utils_data_handling.R | 13 +++++-------- man/interval_long_to_quantile.Rd | 2 +- man/merge_pred_and_obs.Rd | 2 +- man/quantile_to_interval.Rd | 18 +++++++++--------- man/sample_to_interval_long.Rd | 2 +- man/sample_to_quantile.Rd | 2 +- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/R/utils_data_handling.R b/R/utils_data_handling.R index 16896d6a4..2a17689e0 100644 --- a/R/utils_data_handling.R +++ b/R/utils_data_handling.R @@ -16,7 +16,7 @@ #' no ground truth data. #' @param by character vector that denotes the columns by which to merge. Any #' value that is not a column in observations will be removed. -#' @return a data.frame with forecasts and observations +#' @return a data.table with forecasts and observations #' @examples #' forecasts <- example_quantile_forecasts_only #' observations <- example_truth_only @@ -87,13 +87,12 @@ merge_pred_and_obs <- function(forecasts, observations, #' Transform data from a format that is based on predictive samples to a format #' based on plain quantiles. #' -#' #' @param data a data.frame with samples #' @param quantile_level a numeric vector of quantile levels for which #' quantiles will be computed #' @param type type argument passed down to the quantile function. For more #' information, see [quantile()] -#' @return a data.frame in a long interval range format +#' @return a data.table in a long interval range format #' @importFrom data.table as.data.table #' @importFrom stats quantile #' @importFrom methods hasArg @@ -135,7 +134,7 @@ sample_to_quantile <- function(data, #' [score()]) for quantile forecasts. #' @param keep_range_col keep the interval_range and boundary columns after #' transformation (default is FALSE) -#' @return a data.frame in a plain quantile format +#' @return a data.table in a plain quantile format #' @importFrom data.table copy #' @keywords internal @@ -159,7 +158,6 @@ interval_long_to_quantile <- function(data, data[, c("interval_range", "boundary") := NULL] } - return(unique(data)[]) } @@ -183,6 +181,7 @@ interval_long_to_quantile <- function(data, #' and upper bounds of the 50% and 90% prediction intervals (corresponding to #' the 0.25 and 0.75 as well as the 0.05 and 0.095 quantiles). #' @param ... method arguments +#' @return A data.table with forecasts in an interval format #' @keywords data-handling #' @export quantile_to_interval <- function(...) { @@ -201,7 +200,6 @@ quantile_to_interval <- function(...) { #' output after transformation (default is FALSE). This only works if #' `format = "long"`. If `format = "wide"`, the `quantile_level` column will #' always be dropped. -#' @return #' *quantile_to_interval.data.frame*: #' a data.frame in an interval format (either "long" or "wide"), with or #' without a `quantile_level` column. Rows will not be reordered. @@ -243,7 +241,6 @@ quantile_to_interval.data.frame <- function(dt, #' @param predicted a numeric vector of predicted values of size n x N. If #' `observed` is a single number, then `predicted` can be a vector of length N #' @param quantile_level a numeric vector of quantile levels of size N -#' @return #' *quantile_to_interval.numeric*: #' a data.frame in a wide interval format with columns `forecast_id`, #' `observed`, `lower`, `upper`, and `interval_range`. The `forecast_id` column @@ -287,7 +284,7 @@ quantile_to_interval.numeric <- function(observed, #' @param type type argument passed down to the quantile function. For more #' information, see [quantile()] #' @param keep_quantile_col keep quantile_level column, default is TRUE -#' @return a data.frame in a long interval interval range format +#' @return a data.table in a long interval interval range format #' @importFrom data.table as.data.table #' @importFrom stats quantile #' @keywords internal diff --git a/man/interval_long_to_quantile.Rd b/man/interval_long_to_quantile.Rd index b1170f860..1afb06a59 100644 --- a/man/interval_long_to_quantile.Rd +++ b/man/interval_long_to_quantile.Rd @@ -14,7 +14,7 @@ interval_long_to_quantile(data, keep_range_col = FALSE) transformation (default is FALSE)} } \value{ -a data.frame in a plain quantile format +a data.table in a plain quantile format } \description{ Transform data from a format that uses interval ranges to denote quantiles diff --git a/man/merge_pred_and_obs.Rd b/man/merge_pred_and_obs.Rd index 601ff69a9..1c76944b3 100644 --- a/man/merge_pred_and_obs.Rd +++ b/man/merge_pred_and_obs.Rd @@ -26,7 +26,7 @@ no ground truth data.} value that is not a column in observations will be removed.} } \value{ -a data.frame with forecasts and observations +a data.table with forecasts and observations } \description{ The function more or less provides a wrapper around \code{merge} that diff --git a/man/quantile_to_interval.Rd b/man/quantile_to_interval.Rd index 08394367e..8fa0ce4a6 100644 --- a/man/quantile_to_interval.Rd +++ b/man/quantile_to_interval.Rd @@ -27,25 +27,25 @@ prediction interval, respectively.} \item{keep_quantile_col}{keep the \code{quantile_level} column in the final output after transformation (default is FALSE). This only works if \code{format = "long"}. If \code{format = "wide"}, the \code{quantile_level} column will -always be dropped.} +always be dropped. +\emph{quantile_to_interval.data.frame}: +a data.frame in an interval format (either "long" or "wide"), with or +without a \code{quantile_level} column. Rows will not be reordered.} \item{observed}{a numeric vector of observed values of size n} \item{predicted}{a numeric vector of predicted values of size n x N. If \code{observed} is a single number, then \code{predicted} can be a vector of length N} -\item{quantile_level}{a numeric vector of quantile levels of size N} -} -\value{ -\emph{quantile_to_interval.data.frame}: -a data.frame in an interval format (either "long" or "wide"), with or -without a \code{quantile_level} column. Rows will not be reordered. - +\item{quantile_level}{a numeric vector of quantile levels of size N \emph{quantile_to_interval.numeric}: a data.frame in a wide interval format with columns \code{forecast_id}, \code{observed}, \code{lower}, \code{upper}, and \code{interval_range}. The \code{forecast_id} column is a unique identifier for each forecast. Rows will be reordered according to -\code{forecast_id} and \code{interval_range}. +\code{forecast_id} and \code{interval_range}.} +} +\value{ +A data.table with forecasts in an interval format } \description{ \strong{Quantile format} diff --git a/man/sample_to_interval_long.Rd b/man/sample_to_interval_long.Rd index d5029ba2c..cd7f5185e 100644 --- a/man/sample_to_interval_long.Rd +++ b/man/sample_to_interval_long.Rd @@ -23,7 +23,7 @@ information, see \code{\link[=quantile]{quantile()}}} \item{keep_quantile_col}{keep quantile_level column, default is TRUE} } \value{ -a data.frame in a long interval interval range format +a data.table in a long interval interval range format } \description{ Transform data from a format that is based on predictive samples to a format diff --git a/man/sample_to_quantile.Rd b/man/sample_to_quantile.Rd index b16aec18e..4c24f48b9 100644 --- a/man/sample_to_quantile.Rd +++ b/man/sample_to_quantile.Rd @@ -20,7 +20,7 @@ quantiles will be computed} information, see \code{\link[=quantile]{quantile()}}} } \value{ -a data.frame in a long interval range format +a data.table in a long interval range format } \description{ Transform data from a format that is based on predictive samples to a format From 01871401be1b3baf93ddf1009de7de9b26919935 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:42:16 -0600 Subject: [PATCH 18/22] Update return statements for a few utils functions --- R/utils.R | 4 ++-- man/ensure_data.table.Rd | 2 +- man/print.forecast_binary.Rd | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/R/utils.R b/R/utils.R index 088f534a1..5032cfbbd 100644 --- a/R/utils.R +++ b/R/utils.R @@ -113,7 +113,7 @@ run_safely <- function(..., fun) { #' If the object is not a data table, it is converted to one. If the object #' is a data table, a copy of the object is returned. #' @param data An object to ensure is a data table -#' @return A data table +#' @return A data.table/a copy of an exising data.table #' @keywords internal #' @importFrom data.table copy is.data.table as.data.table ensure_data.table <- function(data) { @@ -133,7 +133,7 @@ ensure_data.table <- function(data) { #' @param x An object of class 'forecast_*' object as produced by #' `as_forecast()` #' @param ... additional arguments for [print()] -#' @return NULL +#' @return returns x invisibly #' @export #' @keywords check-forecasts #' @examples diff --git a/man/ensure_data.table.Rd b/man/ensure_data.table.Rd index 6d2457ee5..ef614e8fc 100644 --- a/man/ensure_data.table.Rd +++ b/man/ensure_data.table.Rd @@ -10,7 +10,7 @@ ensure_data.table(data) \item{data}{An object to ensure is a data table} } \value{ -A data table +A data.table/a copy of an exising data.table } \description{ This function ensures that an object is a data table. diff --git a/man/print.forecast_binary.Rd b/man/print.forecast_binary.Rd index 4bbb23c90..60737a0e5 100644 --- a/man/print.forecast_binary.Rd +++ b/man/print.forecast_binary.Rd @@ -21,6 +21,9 @@ \item{...}{additional arguments for \code{\link[=print]{print()}}} } +\value{ +returns x invisibly +} \description{ This function prints information about a forecast object, including "Forecast type", "Score columns", From 89aad154a83f462c8ef40f99cafae73a5308ac92 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:44:38 -0600 Subject: [PATCH 19/22] update return statement --- R/validate.R | 3 +-- man/validate_general.Rd | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/R/validate.R b/R/validate.R index cc5b02c39..3fa98be8f 100644 --- a/R/validate.R +++ b/R/validate.R @@ -228,8 +228,7 @@ validate_forecast.forecast_sample <- function(data, ...) { #' - if appropriate, checks the number of samples / quantiles is the same #' for all forecasts #' @inheritParams get_forecast_counts -#' @return returns the input, with a few new attributes that hold additional -#' information, messages and warnings +#' @return returns the input #' @importFrom data.table ':=' is.data.table #' @importFrom checkmate assert_data_table #' @export diff --git a/man/validate_general.Rd b/man/validate_general.Rd index e379c5a0f..f102b6864 100644 --- a/man/validate_general.Rd +++ b/man/validate_general.Rd @@ -10,8 +10,7 @@ validate_general(data) \item{data}{A data.frame or data.table with predicted and observed values.} } \value{ -returns the input, with a few new attributes that hold additional -information, messages and warnings +returns the input } \description{ The function runs input checks that apply to all input data, regardless of From 79f513adf7be2aeac5b0f61f2c1a9940830d1160 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 00:46:52 -0600 Subject: [PATCH 20/22] Delete `scoringutils-package.R` file which I think didn't do anything... --- NAMESPACE | 1 - R/scoringutils-package.R | 7 ------- man/scoringutils-package.Rd | 14 ++------------ 3 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 R/scoringutils-package.R diff --git a/NAMESPACE b/NAMESPACE index c32f0b1ec..92abdfd84 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -161,7 +161,6 @@ importFrom(ggplot2,theme_minimal) importFrom(ggplot2,unit) importFrom(ggplot2,xlab) importFrom(ggplot2,ylab) -importFrom(lifecycle,deprecated) importFrom(methods,hasArg) importFrom(rlang,warn) importFrom(scoringRules,crps_sample) diff --git a/R/scoringutils-package.R b/R/scoringutils-package.R deleted file mode 100644 index 425b3c1cf..000000000 --- a/R/scoringutils-package.R +++ /dev/null @@ -1,7 +0,0 @@ -#' @keywords internal -"_PACKAGE" - -## usethis namespace: start -#' @importFrom lifecycle deprecated -## usethis namespace: end -NULL diff --git a/man/scoringutils-package.Rd b/man/scoringutils-package.Rd index c41610937..b610ea02b 100644 --- a/man/scoringutils-package.Rd +++ b/man/scoringutils-package.Rd @@ -1,12 +1,11 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/scoringutils-package.R, R/scoringutils.R +% Please edit documentation in R/scoringutils.R \docType{package} \name{scoringutils-package} +\alias{scoringutils} \alias{scoringutils-package} \title{scoringutils: Utilities for Scoring and Assessing Predictions} \description{ -`scoringutils` facilitates the evaluation of forecasts in a convenient framework based on `data.table`. It allows user to to check their forecasts and diagnose issues, to visualise forecasts and missing data, to transform data before scoring, to handle missing forecasts, to aggregate scores, and to visualise the results of the evaluation. The package mostly focuses on the evaluation of probabilistic forecasts and allows evaluating several different forecast types and input formats. Find more information about the package in the Vignettes as well as in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). - `scoringutils` facilitates the evaluation of forecasts in a convenient framework based on `data.table`. It allows user to to check their forecasts and diagnose issues, to visualise forecasts and missing data, to transform data before scoring, to handle missing forecasts, to aggregate scores, and to visualise the results of the evaluation. The package mostly focuses on the evaluation of probabilistic forecasts and allows evaluating several different forecast types and input formats. Find more information about the package in the Vignettes as well as in the accompanying paper (\doi{10.48550/arXiv.2205.07090}). } \seealso{ @@ -18,15 +17,6 @@ Useful links: \item Report bugs at \url{https://github.com/epiforecasts/scoringutils/issues} } - -Useful links: -\itemize{ - \item \doi{10.48550/arXiv.2205.07090} - \item \url{https://epiforecasts.io/scoringutils/} - \item \url{https://github.com/epiforecasts/scoringutils} - \item Report bugs at \url{https://github.com/epiforecasts/scoringutils/issues} -} - } \author{ \strong{Maintainer}: Nikos Bosse \email{nikosbosse@gmail.com} (\href{https://orcid.org/0000-0002-7750-5280}{ORCID}) From 65b892158a583955c98be3f7f2c63b6307ac1810 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 7 Mar 2024 08:37:00 -0500 Subject: [PATCH 21/22] fix linting issue --- R/get_-functions.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/get_-functions.R b/R/get_-functions.R index 97b505008..85adee5cc 100644 --- a/R/get_-functions.R +++ b/R/get_-functions.R @@ -270,8 +270,8 @@ get_protected_columns <- function(data = NULL) { #' get_duplicate_forecasts(example) get_duplicate_forecasts <- function( - data, - forecast_unit = get_forecast_unit(data) + data, + forecast_unit = get_forecast_unit(data) ) { assert_data_frame(data) assert_subset(forecast_unit, colnames(data)) From daf4d5a0721e7b3534c7e7f107438844f174a3c7 Mon Sep 17 00:00:00 2001 From: Nikos Bosse <37978797+nikosbosse@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:28:11 -0500 Subject: [PATCH 22/22] Apply suggestions from code review Co-authored-by: James Azam --- R/available_forecasts.R | 2 +- R/metrics-sample.R | 2 +- R/plot.R | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/available_forecasts.R b/R/available_forecasts.R index 83cfb6c58..b03f4f66d 100644 --- a/R/available_forecasts.R +++ b/R/available_forecasts.R @@ -1,7 +1,7 @@ #' @title Count Number of Available Forecasts #' #' @description -#' Given a data set with forecasts, count the number of available forecasts. +#' Given a data set with forecasts, this function counts the number of available forecasts. #' The level of grouping can be specified using the `by` argument (e.g. to #' count the number of forecasts per model, or the number of forecasts per #' model and location). diff --git a/R/metrics-sample.R b/R/metrics-sample.R index 7cc2c5ec7..27ccd91ae 100644 --- a/R/metrics-sample.R +++ b/R/metrics-sample.R @@ -1,4 +1,4 @@ -#' @title Determines Bias Of Forecasts +#' @title Determine Bias Of Forecasts #' #' @description #' Determines bias from predictive Monte-Carlo samples. The function diff --git a/R/plot.R b/R/plot.R index b35acfb03..ef74f70e2 100644 --- a/R/plot.R +++ b/R/plot.R @@ -665,7 +665,7 @@ plot_pit <- function(pit, #' are shown on the x-axis. #' @param show_counts logical (default is `TRUE`) that indicates whether #' or not to show the actual count numbers on the plot -#' @return ggplot object with a plot of forecast counts +#' @return A ggplot object with a plot of forecast counts #' @importFrom ggplot2 ggplot scale_colour_manual scale_fill_manual #' geom_tile scale_fill_gradient .data #' @importFrom data.table dcast .I .N