feat: add with_test_time (#206)

bin/cargo-bolero/src/random.rs

@@ -19,6 +19,9 @@ pub(crate) fn test(selection: &Selection, test_args: &test::Args) -> Result<()>
     optional_arg!(seed, "BOLERO_RANDOM_SEED");
     optional_arg!(runs, "BOLERO_RANDOM_ITERATIONS");
     optional_arg!(max_input_length, "BOLERO_RANDOM_MAX_LEN");
+    if let Some(t) = test_args.time {
+        cmd.env("BOLERO_RANDOM_TEST_TIME_MS", t.as_millis().to_string());
+    }
 
     // TODO implement other options
     /*

lib/bolero-engine/src/rng.rs

@@ -1,9 +1,11 @@
 use crate::{driver, panic, ByteSliceTestInput, Engine, TargetLocation, Test};
-use core::fmt::Debug;
+use core::{fmt::Debug, time::Duration};
 use rand::{rngs::StdRng, Rng, RngCore, SeedableRng};
+use std::time::Instant;
 
 #[derive(Clone, Copy, Debug)]
 pub struct Options {
+    pub test_time: Option<Duration>,
     pub iterations: Option<usize>,
     pub max_len: Option<usize>,
     pub seed: Option<u64>,
@@ -12,6 +14,7 @@ pub struct Options {
 impl Default for Options {
     fn default() -> Self {
         Self {
+            test_time: get_var("BOLERO_RANDOM_TEST_TIME_MS").map(Duration::from_millis),
             iterations: get_var("BOLERO_RANDOM_ITERATIONS"),
             max_len: get_var("BOLERO_RANDOM_MAX_LEN"),
             seed: get_var("BOLERO_RANDOM_SEED"),
@@ -20,10 +23,18 @@ impl Default for Options {
 }
 
 impl Options {
+    pub fn test_time_or_default(&self) -> Duration {
+        self.test_time.unwrap_or_else(|| {
+            if self.iterations.is_some() {
+                Duration::MAX
+            } else {
+                Duration::from_secs(1)
+            }
+        })
+    }
+
     pub fn iterations_or_default(&self) -> usize {
-        // RNG tests are really slow with miri so we limit the number of iterations
-        self.iterations
-            .unwrap_or(if cfg!(miri) { 25 } else { 1000 })
+        self.iterations.unwrap_or(usize::MAX)
     }
 
     pub fn max_len_or_default(&self) -> usize {
@@ -42,6 +53,7 @@ impl Options {
 /// enough to find edge cases.
 #[derive(Clone)]
 pub struct RngEngine {
+    pub test_time: Duration,
     pub iterations: usize,
     pub max_len: usize,
     pub seed: u64,
@@ -56,6 +68,7 @@ impl Default for RngEngine {
 impl From<Options> for RngEngine {
     fn from(options: Options) -> Self {
         Self {
+            test_time: options.test_time_or_default(),
             iterations: options.iterations_or_default(),
             max_len: options.max_len_or_default(),
             seed: options.seed_or_rand(),
@@ -70,6 +83,11 @@ impl RngEngine {
         Self::default()
     }
 
+    /// Set the test time
+    pub fn with_test_time(self, test_time: Duration) -> Self {
+        Self { test_time, ..self }
+    }
+
     /// Set the number of test iterations
     pub fn with_iterations(self, iterations: usize) -> Self {
         Self { iterations, ..self }
@@ -97,9 +115,10 @@ where
 
         let mut state = RngState::new(self.seed, self.max_len, options);
 
+        let start_time = Instant::now();
         let mut valid = 0;
         let mut invalid = 0;
-        while valid < self.iterations {
+        while valid < self.iterations && start_time.elapsed() < self.test_time {
             match test.test(&mut state.next_input()) {
                 Ok(true) => {
                     valid += 1;
@@ -108,22 +127,8 @@ where
                 Ok(false) => {
                     invalid += 1;
                     if invalid > self.iterations * 2 {
-                        panic!(
-                            concat!(
-                                "Test input could not be satisfied after {} iterations:\n",
-                                "         valid: {}\n",
-                                "       invalid: {}\n",
-                                "  target count: {}\n",
-                                "\n",
-                                "Try reconfiguring the input generator to produce more valid inputs",
-                            ),
-                            valid + invalid,
-                            valid,
-                            invalid,
-                            self.iterations
-                        );
+                        break;
                     }
-                    continue;
                 }
                 #[cfg(not(miri))]
                 Err(_) => {
@@ -147,6 +152,20 @@ where
                 }
             }
         }
+        if invalid > valid * 2 {
+            panic!(
+                concat!(
+                    "Test input generator had too many rejected inputs after {} iterations:\n",
+                    "         valid: {}\n",
+                    "       invalid: {}\n",
+                    "\n",
+                    "Try reconfiguring the input generator to produce more valid inputs",
+                ),
+                valid + invalid,
+                valid,
+                invalid,
+            );
+        }
     }
 }
 

lib/bolero/src/lib.rs

@@ -358,6 +358,12 @@ impl<G: generator::ValueGenerator, Engine, InputOwnership> TestTarget<G, Engine,
 cfg_if::cfg_if! {
     if #[cfg(any(fuzzing, kani))] {
         impl<G, Engine, InputOwnership> TestTarget<G, Engine, InputOwnership> {
+            /// Set the maximum runtime of the tests
+            pub fn with_test_time(self, test_time: Duration) -> Self {
+                let _ = test_time;
+                self
+            }
+
             /// Set the number of iterations executed
             pub fn with_iterations(self, iterations: usize) -> Self {
                 let _ = iterations;
@@ -372,6 +378,12 @@ cfg_if::cfg_if! {
         }
     } else {
         impl<G, InputOwnership> TestTarget<G, crate::test::TestEngine, InputOwnership> {
+            /// Set the maximum runtime of the tests
+            pub fn with_test_time(mut self, test_time: Duration) -> Self {
+                self.engine.with_test_time(test_time);
+                self
+            }
+
             /// Set the number of iterations executed
             pub fn with_iterations(mut self, iterations: usize) -> Self {
                 self.engine.with_iterations(iterations);

lib/bolero/src/test/mod.rs

@@ -1,8 +1,8 @@
 #![cfg_attr(fuzzing_random, allow(dead_code))]
 
 use bolero_engine::{driver, rng, test_failure::TestFailure, Engine, TargetLocation, Test};
-use core::iter::empty;
-use std::path::PathBuf;
+use core::{iter::empty, time::Duration};
+use std::{path::PathBuf, time::Instant};
 
 mod input;
 use input::*;
@@ -33,6 +33,11 @@ impl TestEngine {
         }
     }
 
+    pub fn with_test_time(&mut self, test_time: Duration) -> &mut Self {
+        self.rng_cfg.test_time = self.rng_cfg.test_time.or(Some(test_time));
+        self
+    }
+
     pub fn with_iterations(&mut self, iterations: usize) -> &mut Self {
         self.rng_cfg.iterations = self.rng_cfg.iterations.or(Some(iterations));
         self
@@ -90,7 +95,7 @@ impl TestEngine {
             .map(move |seed| input::RngTest { seed, max_len })
     }
 
-    fn tests(&self) -> Vec<NamedTest> {
+    fn tests(&self) -> impl Iterator<Item = NamedTest> {
         let rng_tests = self.rng_tests().map(move |test| NamedTest {
             name: format!("[BOLERO_RANDOM_SEED={}]", test.seed),
             data: TestInput::RngTest(test),
@@ -103,7 +108,6 @@ impl TestEngine {
             .chain(self.file_tests(["corpus"].iter().cloned()))
             .chain(self.file_tests(["afl_state", "queue"].iter().cloned()))
             .chain(rng_tests)
-            .collect()
     }
 
     #[cfg(any(fuzzing_random, test))]
@@ -246,7 +250,13 @@ impl TestEngine {
         bolero_engine::panic::set_hook();
         bolero_engine::panic::forward_panic(false);
 
+        let start_time = Instant::now();
+        let test_time = self.rng_cfg.test_time_or_default();
         for test in tests {
+            if start_time.elapsed() > test_time {
+                break;
+            }
+
             progress();
 
             if let Err(err) = testfn(&test.data) {

lib/bolero/src/tests.rs

@@ -53,7 +53,7 @@ fn range_generator_cloned_test() {
 #[test]
 fn nested_test() {
     check!().with_generator(0..=5).for_each(|_input: &u8| {
-        // println!("{:?}", input);
+        // println!("{:?}", _input);
     });
 }
 
@@ -67,3 +67,16 @@ fn iteration_number() {
     });
     assert_eq!(num_iters.load(Ordering::Relaxed), 5);
 }
+
+#[test]
+fn with_test_time() {
+    // Atomic to avoid having to think about unwind safety
+    use std::sync::atomic::Ordering;
+    let num_iters = std::sync::atomic::AtomicUsize::new(0);
+    check!()
+        .with_test_time(core::time::Duration::from_millis(5))
+        .for_each(|_| {
+            num_iters.fetch_add(1, Ordering::Relaxed);
+        });
+    assert!(num_iters.load(Ordering::Relaxed) > 10);
+}