-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #122 from voltrondata-labs/edward/top-level-runner
Add a top-level runner for sets of benchmarks
- Loading branch information
Showing
18 changed files
with
459 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
#' A classed dataframe of benchmarks for running | ||
#' | ||
#' @param benchmarks A list with elements of class `Benchmark` | ||
#' @param parameters Optional. A list of dataframes of parameter combinations to | ||
#' run as generated by [get_default_parameters()]. If null, defaults will be generated | ||
#' when [run()] is called. | ||
#' | ||
#' @return A classed dataframe with `name` (benchmark attribute, not object name), | ||
#' `benchmark`, and `params` columns | ||
#' | ||
#' @export | ||
BenchmarkDataFrame <- function(benchmarks, parameters) { | ||
lapply(benchmarks, function(bm) stopifnot( | ||
"All elements of `benchmarks` are not of class `Benchmark`!" = inherits(bm, "Benchmark") | ||
)) | ||
|
||
bm_names <- vapply(benchmarks, function(bm) bm$name, character(1)) | ||
if (missing(parameters)) { | ||
parameters <- rep(list(NULL), length = length(benchmarks)) | ||
} | ||
|
||
structure( | ||
tibble::tibble( | ||
name = bm_names, | ||
benchmark = benchmarks, | ||
parameters = parameters | ||
), | ||
class = c("BenchmarkDataFrame", "tbl_df", "tbl", "data.frame") | ||
) | ||
} | ||
|
||
|
||
#' @export | ||
format.BenchmarkDataFrame <- function(x, ...) { | ||
c("# <BenchmarkDataFrame>", NextMethod()) | ||
} | ||
|
||
|
||
#' Get a list of benchmarks in a package | ||
#' | ||
#' @param package String of package name in which to find benchmarks | ||
#' | ||
#' @return An instance of [BenchmarkDataFrame] with all the benchmarks contained | ||
#' by a package | ||
#' | ||
#' @export | ||
get_package_benchmarks <- function(package = "arrowbench") { | ||
nms <- getNamespaceExports(package) | ||
objs <- mget(nms, envir = getNamespace(package)) | ||
bms <- Filter(function(x) inherits(x, "Benchmark"), objs) | ||
BenchmarkDataFrame(benchmarks = bms) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
#' Generate a dataframe of default parameters for a benchmark | ||
#' | ||
#' Generates a dataframe of parameter combinations for a benchmark to try based | ||
#' on the parameter defaults of its `setup` function and supplied parameters. | ||
#' | ||
#' @param x An object for which to generate parameters | ||
#' @param ... Named arguments corresponding to the parameters of `bm`'s `setup` | ||
#' function. May also contain `cpu_count`, `lib_path`, and `mem_alloc`. | ||
#' | ||
#' @return For `get_default_parameters.Benchmark`, a dataframe of parameter combinations | ||
#' to try with a column for each parameter and a row for each combination. | ||
#' | ||
#' @export | ||
get_default_parameters <- function(x, ...) { | ||
UseMethod("get_default_parameters") | ||
} | ||
|
||
#' @rdname get_default_parameters | ||
#' @export | ||
get_default_parameters.default <- function(x, ...) { | ||
stop("No method found for class `", toString(class(x)), '`') | ||
} | ||
|
||
#' @rdname get_default_parameters | ||
#' @export | ||
get_default_parameters.Benchmark <- function(x, ...) { | ||
# This takes the expansion of the default parameters in the function signature | ||
# perhaps restricted by the ... params | ||
params <- modifyList(get_default_args(x$setup), list(...)) | ||
if (identical(params$lib_path, "all")) { | ||
# Default for lib_path is just "latest", if omitted | ||
# "all" means all old versions | ||
# rev() is so we run newest first. This also means we bootstrap data fixtures | ||
# with newest first, so that's some assurance that older versions can read | ||
# what the newer libs write | ||
params$lib_path <- rev(c(names(arrow_version_to_date), "devel", "latest")) | ||
} | ||
if (is.null(params$cpu_count)) { | ||
params$cpu_count <- c(1L, parallel::detectCores()) | ||
} | ||
params$stringsAsFactors <- FALSE | ||
out <- do.call(expand.grid, params) | ||
|
||
# we don't change memory allocators on non-arrow packages | ||
if (!is.null(params$mem_alloc)) { | ||
# a bit of a hack, we can test memory allocators on devel or latest, but | ||
# "4.0" <= "devel" and "4.0" <= "latest" are both true. | ||
out[!is_arrow_package(out, "4.0", x$packages_used), "mem_alloc"] <- NA | ||
out <- unique(out) | ||
} | ||
|
||
if (!is.null(x$valid_params)) { | ||
out <- x$valid_params(out) | ||
} | ||
out | ||
} | ||
|
||
#' @rdname get_default_parameters | ||
#' @export | ||
get_default_parameters.BenchmarkDataFrame <- function(x, ...) { | ||
x$parameters <- purrr::map2(x$benchmark, x$parameters, function(bm, params) { | ||
if (is.null(params)) { | ||
params <- get_default_parameters(bm, ...) | ||
} | ||
params | ||
}) | ||
|
||
x | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.