library/test/src/bench.rs

   1 //! Benchmarking module.
   2 pub use std::hint::black_box;
   3
   4 use super::{
   5     event::CompletedTest, helpers::sink::Sink, options::BenchMode, test_result::TestResult,
   6     types::TestDesc, Sender,
   7 };
   8
   9 use crate::stats;
  10 use std::cmp;
  11 use std::io;
  12 use std::panic::{catch_unwind, AssertUnwindSafe};
  13 use std::sync::{Arc, Mutex};
  14 use std::time::{Duration, Instant};
  15
  16 /// Manager of the benchmarking runs.
  17 ///
  18 /// This is fed into functions marked with `#[bench]` to allow for
  19 /// set-up & tear-down before running a piece of code repeatedly via a
  20 /// call to `iter`.
  21 #[derive(Clone)]
  22 pub struct Bencher {
  23     mode: BenchMode,
  24     summary: Option<stats::Summary>,
  25     pub bytes: u64,
  26 }
  27
  28 impl Bencher {
  29     /// Callback for benchmark functions to run in their body.
  30     pub fn iter<T, F>(&mut self, mut inner: F)
  31     where
  32         F: FnMut() -> T,
  33     {
  34         if self.mode == BenchMode::Single {
  35             ns_iter_inner(&mut inner, 1);
  36             return;
  37         }
  38
  39         self.summary = Some(iter(&mut inner));
  40     }
  41
  42     pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary>
  43     where
  44         F: FnMut(&mut Bencher),
  45     {
  46         f(self);
  47         self.summary
  48     }
  49 }
  50
  51 #[derive(Debug, Clone, PartialEq)]
  52 pub struct BenchSamples {
  53     pub ns_iter_summ: stats::Summary,
  54     pub mb_s: usize,
  55 }
  56
  57 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
  58     use std::fmt::Write;
  59     let mut output = String::new();
  60
  61     let median = bs.ns_iter_summ.median as usize;
  62     let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
  63
  64     write!(
  65         output,
  66         "{:>11} ns/iter (+/- {})",
  67         fmt_thousands_sep(median, ','),
  68         fmt_thousands_sep(deviation, ',')
  69     )
  70     .unwrap();
  71     if bs.mb_s != 0 {
  72         write!(output, " = {} MB/s", bs.mb_s).unwrap();
  73     }
  74     output
  75 }
  76
  77 // Format a number with thousands separators
  78 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
  79     use std::fmt::Write;
  80     let mut output = String::new();
  81     let mut trailing = false;
  82     for &pow in &[9, 6, 3, 0] {
  83         let base = 10_usize.pow(pow);
  84         if pow == 0 || trailing || n / base != 0 {
  85             if !trailing {
  86                 write!(output, "{}", n / base).unwrap();
  87             } else {
  88                 write!(output, "{:03}", n / base).unwrap();
  89             }
  90             if pow != 0 {
  91                 output.push(sep);
  92             }
  93             trailing = true;
  94         }
  95         n %= base;
  96     }
  97
  98     output
  99 }
 100
 101 fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64
 102 where
 103     F: FnMut() -> T,
 104 {
 105     let start = Instant::now();
 106     for _ in 0..k {
 107         black_box(inner());
 108     }
 109     start.elapsed().as_nanos() as u64
 110 }
 111
 112 pub fn iter<T, F>(inner: &mut F) -> stats::Summary
 113 where
 114     F: FnMut() -> T,
 115 {
 116     // Initial bench run to get ballpark figure.
 117     let ns_single = ns_iter_inner(inner, 1);
 118
 119     // Try to estimate iter count for 1ms falling back to 1m
 120     // iterations if first run took < 1ns.
 121     let ns_target_total = 1_000_000; // 1ms
 122     let mut n = ns_target_total / cmp::max(1, ns_single);
 123
 124     // if the first run took more than 1ms we don't want to just
 125     // be left doing 0 iterations on every loop. The unfortunate
 126     // side effect of not being able to do as many runs is
 127     // automatically handled by the statistical analysis below
 128     // (i.e., larger error bars).
 129     n = cmp::max(1, n);
 130
 131     let mut total_run = Duration::new(0, 0);
 132     let samples: &mut [f64] = &mut [0.0_f64; 50];
 133     loop {
 134         let loop_start = Instant::now();
 135
 136         for p in &mut *samples {
 137             *p = ns_iter_inner(inner, n) as f64 / n as f64;
 138         }
 139
 140         stats::winsorize(samples, 5.0);
 141         let summ = stats::Summary::new(samples);
 142
 143         for p in &mut *samples {
 144             let ns = ns_iter_inner(inner, 5 * n);
 145             *p = ns as f64 / (5 * n) as f64;
 146         }
 147
 148         stats::winsorize(samples, 5.0);
 149         let summ5 = stats::Summary::new(samples);
 150
 151         let loop_run = loop_start.elapsed();
 152
 153         // If we've run for 100ms and seem to have converged to a
 154         // stable median.
 155         if loop_run > Duration::from_millis(100)
 156             && summ.median_abs_dev_pct < 1.0
 157             && summ.median - summ5.median < summ5.median_abs_dev
 158         {
 159             return summ5;
 160         }
 161
 162         total_run += loop_run;
 163         // Longest we ever run for is 3s.
 164         if total_run > Duration::from_secs(3) {
 165             return summ5;
 166         }
 167
 168         // If we overflow here just return the results so far. We check a
 169         // multiplier of 10 because we're about to multiply by 2 and the
 170         // next iteration of the loop will also multiply by 5 (to calculate
 171         // the summ5 result)
 172         n = match n.checked_mul(10) {
 173             Some(_) => n * 2,
 174             None => {
 175                 return summ5;
 176             }
 177         };
 178     }
 179 }
 180
 181 pub fn benchmark<F>(desc: TestDesc, monitor_ch: Sender<CompletedTest>, nocapture: bool, f: F)
 182 where
 183     F: FnMut(&mut Bencher),
 184 {
 185     let mut bs = Bencher { mode: BenchMode::Auto, summary: None, bytes: 0 };
 186
 187     let data = Arc::new(Mutex::new(Vec::new()));
 188     let oldio = if !nocapture {
 189         Some((
 190             io::set_print(Some(Sink::new_boxed(&data))),
 191             io::set_panic(Some(Sink::new_boxed(&data))),
 192         ))
 193     } else {
 194         None
 195     };
 196
 197     let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f)));
 198
 199     if let Some((printio, panicio)) = oldio {
 200         io::set_print(printio);
 201         io::set_panic(panicio);
 202     }
 203
 204     let test_result = match result {
 205         //bs.bench(f) {
 206         Ok(Some(ns_iter_summ)) => {
 207             let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
 208             let mb_s = bs.bytes * 1000 / ns_iter;
 209
 210             let bs = BenchSamples { ns_iter_summ, mb_s: mb_s as usize };
 211             TestResult::TrBench(bs)
 212         }
 213         Ok(None) => {
 214             // iter not called, so no data.
 215             // FIXME: error in this case?
 216             let samples: &mut [f64] = &mut [0.0_f64; 1];
 217             let bs = BenchSamples { ns_iter_summ: stats::Summary::new(samples), mb_s: 0 };
 218             TestResult::TrBench(bs)
 219         }
 220         Err(_) => TestResult::TrFailed,
 221     };
 222
 223     let stdout = data.lock().unwrap().to_vec();
 224     let message = CompletedTest::new(desc, test_result, None, stdout);
 225     monitor_ch.send(message).unwrap();
 226 }
 227
 228 pub fn run_once<F>(f: F)
 229 where
 230     F: FnMut(&mut Bencher),
 231 {
 232     let mut bs = Bencher { mode: BenchMode::Single, summary: None, bytes: 0 };
 233     bs.bench(f);
 234 }