Merge pull request #122 from voltrondata-labs/edward/top-level-runner

Add a top-level runner for sets of benchmarks

DESCRIPTION

@@ -15,8 +15,6 @@ Depends: R (>= 3.5.0)
 Imports:
     arrow,
     bench,
-    callr,
-    dbplyr,
     dplyr,
     duckdb (>= 0.6.0),
     distro,
@@ -29,6 +27,7 @@ Imports:
     rlang,
     R.utils,
     sessioninfo,
+    tibble,
     utils,
     uuid,
     waldo,
@@ -38,6 +37,7 @@ Suggests:
     archive,
     data.table,
     DBI,
+    dbplyr,
     fst,
     jsonify,
     lubridate,
@@ -49,6 +49,7 @@ Suggests:
 RoxygenNote: 7.2.2
 Roxygen: list(markdown = TRUE, load = "source")
 Collate: 
+    'benchmark-dataframe.R'
     'benchmark.R'
     'bm-array-altrep-materialization.R'
     'bm-array-to-vector.R'
@@ -72,6 +73,7 @@ Collate:
     'ensure-source.R'
     'ensure-tpch-source.R'
     'measure.R'
+    'params.R'
     'util.R'
     'result.R'
     'run.R'

NAMESPACE

@@ -4,9 +4,16 @@ S3method(as.character,Serializable)
 S3method(as.data.frame,BenchmarkResult)
 S3method(as.data.frame,BenchmarkResults)
 S3method(as.list,Serializable)
+S3method(format,BenchmarkDataFrame)
+S3method(get_default_parameters,Benchmark)
+S3method(get_default_parameters,BenchmarkDataFrame)
+S3method(get_default_parameters,default)
+S3method(run,BenchmarkDataFrame)
+S3method(run,default)
 export("%||%")
 export(BenchEnvironment)
 export(Benchmark)
+export(BenchmarkDataFrame)
 export(all_sources)
 export(array_altrep_materialization)
 export(array_to_vector)
@@ -22,8 +29,10 @@ export(generate_tpch)
 export(get_csv_reader)
 export(get_csv_writer)
 export(get_dataset_attr)
+export(get_default_parameters)
 export(get_input_func)
 export(get_json_reader)
+export(get_package_benchmarks)
 export(get_params_summary)
 export(get_query_func)
 export(get_read_function)
@@ -40,6 +49,7 @@ export(read_json)
 export(read_source)
 export(remote_dataset)
 export(row_group_size)
+export(run)
 export(run_benchmark)
 export(run_bm)
 export(run_one)

R/benchmark-dataframe.R

@@ -0,0 +1,52 @@
+#' A classed dataframe of benchmarks for running
+#'
+#' @param benchmarks A list with elements of class `Benchmark`
+#' @param parameters Optional. A list of dataframes of parameter combinations to
+#' run as generated by [get_default_parameters()]. If null, defaults will be generated
+#' when [run()] is called.
+#'
+#' @return A classed dataframe with `name` (benchmark attribute, not object name),
+#' `benchmark`, and `params` columns
+#'
+#' @export
+BenchmarkDataFrame <- function(benchmarks, parameters) {
+  lapply(benchmarks, function(bm) stopifnot(
+    "All elements of `benchmarks` are not of class `Benchmark`!" = inherits(bm, "Benchmark")
+  ))
+
+  bm_names <- vapply(benchmarks, function(bm) bm$name, character(1))
+  if (missing(parameters)) {
+    parameters <- rep(list(NULL), length = length(benchmarks))
+  }
+
+  structure(
+    tibble::tibble(
+      name = bm_names,
+      benchmark = benchmarks,
+      parameters = parameters
+    ),
+    class = c("BenchmarkDataFrame", "tbl_df", "tbl", "data.frame")
+  )
+}
+
+
+#' @export
+format.BenchmarkDataFrame <- function(x, ...) {
+  c("# <BenchmarkDataFrame>", NextMethod())
+}
+
+
+#' Get a list of benchmarks in a package
+#'
+#' @param package String of package name in which to find benchmarks
+#'
+#' @return An instance of [BenchmarkDataFrame] with all the benchmarks contained
+#' by a package
+#'
+#' @export
+get_package_benchmarks <- function(package = "arrowbench") {
+  nms <- getNamespaceExports(package)
+  objs <- mget(nms, envir = getNamespace(package))
+  bms <- Filter(function(x) inherits(x, "Benchmark"), objs)
+  BenchmarkDataFrame(benchmarks = bms)
+}

R/benchmark.R

@@ -129,44 +129,14 @@ Benchmark <- function(name,
   )
 }
 
+
 #' Create a test environment to run benchmarks in
 #'
 #' @param ... named list of parameters to set in the environment
 #' @return An environment
 #' @export
 BenchEnvironment <- function(...) list2env(list(...))
 
-default_params <- function(bm, ...) {
-  # This takes the expansion of the default parameters in the function signature
-  # perhaps restricted by the ... params
-  params <- modifyList(get_default_args(bm$setup), list(...))
-  if (identical(params$lib_path, "all")) {
-    # Default for lib_path is just "latest", if omitted
-    # "all" means all old versions
-    # rev() is so we run newest first. This also means we bootstrap data fixtures
-    # with newest first, so that's some assurance that older versions can read
-    # what the newer libs write
-    params$lib_path <- rev(c(names(arrow_version_to_date), "devel", "latest"))
-  }
-  if (is.null(params$cpu_count)) {
-    params$cpu_count <- c(1L, parallel::detectCores())
-  }
-  params$stringsAsFactors <- FALSE
-  out <- do.call(expand.grid, params)
-
-  # we don't change memory allocators on non-arrow packages
-  if (!is.null(params$mem_alloc)) {
-    # a bit of a hack, we can test memory allocators on devel or latest, but
-    # "4.0" <= "devel" and "4.0" <= "latest" are both true.
-    out[!is_arrow_package(out, "4.0", bm$packages_used), "mem_alloc"] <- NA
-    out <- unique(out)
-  }
-
-  if (!is.null(bm$valid_params)) {
-    out <- bm$valid_params(out)
-  }
-  out
-}
 
 #' Extract the parameter summary as a data.frame
 #'

R/params.R

@@ -0,0 +1,69 @@
+#' Generate a dataframe of default parameters for a benchmark
+#'
+#' Generates a dataframe of parameter combinations for a benchmark to try based
+#' on the parameter defaults of its `setup` function and supplied parameters.
+#'
+#' @param x An object for which to generate parameters
+#' @param ... Named arguments corresponding to the parameters of `bm`'s `setup`
+#' function. May also contain `cpu_count`, `lib_path`, and `mem_alloc`.
+#'
+#' @return For `get_default_parameters.Benchmark`, a dataframe of parameter combinations
+#' to try with a column for each parameter and a row for each combination.
+#'
+#' @export
+get_default_parameters <- function(x, ...) {
+  UseMethod("get_default_parameters")
+}
+
+#' @rdname get_default_parameters
+#' @export
+get_default_parameters.default <- function(x, ...) {
+  stop("No method found for class `", toString(class(x)), '`')
+}
+
+#' @rdname get_default_parameters
+#' @export
+get_default_parameters.Benchmark <- function(x, ...) {
+  # This takes the expansion of the default parameters in the function signature
+  # perhaps restricted by the ... params
+  params <- modifyList(get_default_args(x$setup), list(...))
+  if (identical(params$lib_path, "all")) {
+    # Default for lib_path is just "latest", if omitted
+    # "all" means all old versions
+    # rev() is so we run newest first. This also means we bootstrap data fixtures
+    # with newest first, so that's some assurance that older versions can read
+    # what the newer libs write
+    params$lib_path <- rev(c(names(arrow_version_to_date), "devel", "latest"))
+  }
+  if (is.null(params$cpu_count)) {
+    params$cpu_count <- c(1L, parallel::detectCores())
+  }
+  params$stringsAsFactors <- FALSE
+  out <- do.call(expand.grid, params)
+
+  # we don't change memory allocators on non-arrow packages
+  if (!is.null(params$mem_alloc)) {
+    # a bit of a hack, we can test memory allocators on devel or latest, but
+    # "4.0" <= "devel" and "4.0" <= "latest" are both true.
+    out[!is_arrow_package(out, "4.0", x$packages_used), "mem_alloc"] <- NA
+    out <- unique(out)
+  }
+
+  if (!is.null(x$valid_params)) {
+    out <- x$valid_params(out)
+  }
+  out
+}
+
+#' @rdname get_default_parameters
+#' @export
+get_default_parameters.BenchmarkDataFrame <- function(x, ...) {
+  x$parameters <- purrr::map2(x$benchmark, x$parameters, function(bm, params) {
+    if (is.null(params)) {
+      params <- get_default_parameters(bm, ...)
+    }
+    params
+  })
+
+  x
+}

R/run.R

@@ -1,3 +1,38 @@
+#' Run an object
+#'
+#' @param x An S3 classed object to run
+#' @param ... Additional arguments passed through to methods. For
+#' `run.BenchmarkDataFrame`, passed through to [get_default_parameters()] (when
+#' parameters are not specified) and [run_benchmark()].
+#'
+#' @return A modified object containing run results. For `run.BenchmarkDataFrame`,
+#' a `results` list column is appended.
+#'
+#' @export
+run <- function(x, ...) {
+  UseMethod("run")
+}
+
+
+#' @export
+run.default <- function(x, ...) {
+  stop("No method found for class `", toString(class(x)), '`')
+}
+
+
+#' @rdname run
+#' @export
+run.BenchmarkDataFrame <- function(x, ...) {
+  # if already run (so no elements of `parameters` are NULL), is no-op
+  x <- get_default_parameters(x, ...)
+
+  x$results <- purrr::map2(x$benchmark, x$parameters, function(bm, params) {
+    run_benchmark(bm = bm, params = params, ...)
+  })
+
+  x
+}
+
 #' Run a Benchmark across a range of parameters
 #'
 #' @param bm [Benchmark()] object
@@ -23,7 +58,7 @@
 #' @importFrom progress progress_bar
 run_benchmark <- function(bm,
                           ...,
-                          params = default_params(bm, ...),
+                          params = get_default_parameters(bm, ...),
                           n_iter = 1,
                           dry_run = FALSE,
                           profiling = FALSE,

README.md

@@ -229,6 +229,44 @@ run_benchmark(
 )
 ```
 
+### Running a set of benchmarks together
+
+The `get_package_benchmarks()` function gets all the benchmarks defined in a 
+package—by default this one—and returns them in a classed dataframe with columns 
+for the benchmark name attribute, the benchmark itself (in a list column), and 
+a list column of dataframes of parameters with which to run the benchmark (NULL 
+by default, meaning use `get_default_parameters(benchmark)` for each:
+
+``` r
+> get_package_benchmarks()
+# <BenchmarkDataFrame>
+# A tibble: 14 × 3
+   name                         benchmark    parameters
+ * <chr>                        <named list> <list>    
+ 1 array_to_vector              <Benchmrk>   <NULL>    
+ 2 remote_dataset               <Benchmrk>   <NULL>    
+ 3 row_group_size               <Benchmrk>   <NULL>    
+ 4 file-read                    <Benchmrk>   <NULL>    
+ 5 dataframe-to-table           <Benchmrk>   <NULL>    
+ 6 write_csv                    <Benchmrk>   <NULL>    
+ 7 array_altrep_materialization <Benchmrk>   <NULL>    
+ 8 partitioned-dataset-filter   <Benchmrk>   <NULL>    
+ 9 read_csv                     <Benchmrk>   <NULL>    
+10 read_json                    <Benchmrk>   <NULL>    
+11 tpch                         <Benchmrk>   <NULL>    
+12 dataset_taxi_2013            <Benchmrk>   <NULL>    
+13 file-write                   <Benchmrk>   <NULL>    
+14 table_to_df                  <Benchmrk>   <NULL> 
+```
+
+If certain benchmarks are to be run on certain machines, the dataframe can be 
+subset with normal dataframe operations. If parameters other than defaults 
+should be used, the `parameters` column can be filled in manually. When ready, 
+the dataframe can be passed to `run()`, which will run each benchmark on each of 
+its sets of parameters and append a `results` column to the returned dataset 
+that contains result objects that can be transformed to JSON appropriate for 
+sending to a Conbench server.
+
 ### Enabling benchmarks to be run on conbench
 
 [Conbench](https://conbench.ursa.dev/) is a service that runs benchmarks continuously on a repo. We have a conbench