storage_benchmarks/
lib.rs

1// Copyright 2022 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5pub mod block_device;
6pub mod directory_benchmarks;
7pub mod filesystem;
8pub mod io_benchmarks;
9pub mod testing;
10
11use async_trait::async_trait;
12use fuchsiaperf::FuchsiaPerfBenchmarkResult;
13use log::info;
14use regex::RegexSet;
15use std::io::Write;
16use std::time::Instant;
17
18pub use crate::block_device::{BlockDeviceConfig, BlockDeviceFactory};
19pub use crate::filesystem::{CacheClearableFilesystem, Filesystem, FilesystemConfig};
20
21/// How long a benchmarked operation took to complete.
22#[derive(Debug)]
23pub struct OperationDuration(u64);
24
25/// A timer for tracking how long an operation took to complete.
26#[derive(Debug)]
27pub struct OperationTimer {
28    start_time: Instant,
29}
30
31impl OperationTimer {
32    pub fn start() -> Self {
33        let start_time = Instant::now();
34        Self { start_time }
35    }
36
37    pub fn stop(self) -> OperationDuration {
38        OperationDuration(Instant::now().duration_since(self.start_time).as_nanos() as u64)
39    }
40}
41
42/// A trait representing a single benchmark.
43#[async_trait]
44pub trait Benchmark<T>: Send + Sync {
45    async fn run(&self, fs: &mut T) -> Vec<OperationDuration>;
46    fn name(&self) -> String;
47}
48
49struct BenchmarkResultsStatistics {
50    p50: u64,
51    p95: u64,
52    p99: u64,
53    mean: u64,
54    min: u64,
55    max: u64,
56    count: u64,
57}
58
59struct BenchmarkResults {
60    benchmark_name: String,
61    filesystem_name: String,
62    values: Vec<OperationDuration>,
63}
64
65fn percentile(data: &[u64], p: u8) -> u64 {
66    assert!(p <= 100 && !data.is_empty());
67    if data.len() == 1 || p == 0 {
68        return *data.first().unwrap();
69    }
70    if p == 100 {
71        return *data.last().unwrap();
72    }
73    let rank = ((p as f64) / 100.0) * ((data.len() - 1) as f64);
74    // The rank is unlikely to be a whole number. Use a linear interpolation between the 2 closest
75    // data points.
76    let fraction = rank - rank.floor();
77    let rank = rank as usize;
78    data[rank] + ((data[rank + 1] - data[rank]) as f64 * fraction).round() as u64
79}
80
81impl BenchmarkResults {
82    fn statistics(&self) -> BenchmarkResultsStatistics {
83        let mut data: Vec<u64> = self.values.iter().map(|d| d.0).collect();
84        data.sort_by(|a, b| a.partial_cmp(b).unwrap());
85        let min = *data.first().unwrap();
86        let max = *data.last().unwrap();
87        let p50 = percentile(&data, 50);
88        let p95 = percentile(&data, 95);
89        let p99 = percentile(&data, 99);
90
91        let mut sum = 0;
92        for value in &data {
93            sum += value;
94        }
95        let count = data.len() as u64;
96        let mean = sum / count;
97        BenchmarkResultsStatistics { min, p50, p95, p99, max, mean, count }
98    }
99
100    fn pretty_row_header() -> prettytable::Row {
101        use prettytable::{cell, row};
102        row![
103            l->"FS",
104            l->"Benchmark",
105            r->"Min",
106            r->"50th Percentile",
107            r->"95th Percentile",
108            r->"99th Percentile",
109            r->"Max",
110            r->"Mean",
111            r->"Count"
112        ]
113    }
114
115    fn pretty_row(&self) -> prettytable::Row {
116        let stats = self.statistics();
117        use prettytable::{cell, row};
118        row![
119            l->self.filesystem_name,
120            l->self.benchmark_name,
121            r->format_u64_with_commas(stats.min),
122            r->format_u64_with_commas(stats.p50),
123            r->format_u64_with_commas(stats.p95),
124            r->format_u64_with_commas(stats.p99),
125            r->format_u64_with_commas(stats.max),
126            r->format_u64_with_commas(stats.mean),
127            r->format_u64_with_commas(stats.count),
128        ]
129    }
130
131    fn csv_row_header() -> String {
132        "FS,Benchmark,Min,50th Percentile,95th Percentile,99th Percentile,Max,Mean,Count\n"
133            .to_owned()
134    }
135
136    fn csv_row(&self) -> String {
137        let stats = self.statistics();
138        format!(
139            "{},{},{},{},{},{},{},{},{}\n",
140            self.filesystem_name,
141            self.benchmark_name,
142            stats.min,
143            stats.p50,
144            stats.p95,
145            stats.p99,
146            stats.max,
147            stats.mean,
148            stats.count
149        )
150    }
151}
152
153#[async_trait]
154trait BenchmarkConfig {
155    async fn run(&self, block_device_factory: &dyn BlockDeviceFactory) -> BenchmarkResults;
156    fn name(&self) -> String;
157    fn matches(&self, filter: &RegexSet) -> bool;
158}
159
160struct BenchmarkConfigImpl<T, U>
161where
162    T: Benchmark<U::Filesystem>,
163    U: FilesystemConfig,
164{
165    benchmark: T,
166    filesystem_config: U,
167}
168
169#[async_trait]
170impl<T, U> BenchmarkConfig for BenchmarkConfigImpl<T, U>
171where
172    T: Benchmark<U::Filesystem>,
173    U: FilesystemConfig,
174{
175    async fn run(&self, block_device_factory: &dyn BlockDeviceFactory) -> BenchmarkResults {
176        let mut fs = self.filesystem_config.start_filesystem(block_device_factory).await;
177        info!("Running {}", self.name());
178        let timer = OperationTimer::start();
179        let durations = self.benchmark.run(&mut fs).await;
180        let benchmark_duration = timer.stop();
181        info!("Finished {} {}ns", self.name(), format_u64_with_commas(benchmark_duration.0));
182        fs.shutdown().await;
183
184        BenchmarkResults {
185            benchmark_name: self.benchmark.name(),
186            filesystem_name: self.filesystem_config.name(),
187            values: durations,
188        }
189    }
190
191    fn name(&self) -> String {
192        format!("{}/{}", self.benchmark.name(), self.filesystem_config.name())
193    }
194
195    fn matches(&self, filter: &RegexSet) -> bool {
196        if filter.is_empty() {
197            return true;
198        }
199        filter.is_match(&self.name())
200    }
201}
202
203/// A collection of benchmarks and the filesystems to run the benchmarks against.
204pub struct BenchmarkSet {
205    benchmarks: Vec<Box<dyn BenchmarkConfig>>,
206}
207
208impl BenchmarkSet {
209    pub fn new() -> Self {
210        Self { benchmarks: Vec::new() }
211    }
212
213    /// Adds a new benchmark with the filesystem it should be run against to the `BenchmarkSet`.
214    pub fn add_benchmark<T, U>(&mut self, benchmark: T, filesystem_config: U)
215    where
216        T: Benchmark<U::Filesystem> + 'static,
217        U: FilesystemConfig + 'static,
218    {
219        self.benchmarks.push(Box::new(BenchmarkConfigImpl { benchmark, filesystem_config }));
220    }
221
222    /// Runs all of the added benchmarks against their configured filesystems. The filesystems will
223    /// be brought up on block devices created from `block_device_factory`.
224    pub async fn run(
225        &self,
226        block_device_factory: &dyn BlockDeviceFactory,
227        filter: &RegexSet,
228    ) -> BenchmarkSetResults {
229        let mut results = Vec::new();
230        for benchmark in &self.benchmarks {
231            if benchmark.matches(filter) {
232                results.push(benchmark.run(block_device_factory).await);
233            }
234        }
235        BenchmarkSetResults { results }
236    }
237}
238
239/// All of the benchmark results from `BenchmarkSet::run`.
240pub struct BenchmarkSetResults {
241    results: Vec<BenchmarkResults>,
242}
243
244impl BenchmarkSetResults {
245    /// Writes the benchmark results in the fuchsiaperf.json format to `writer`.
246    pub fn write_fuchsia_perf_json<F: Write>(&self, writer: F) {
247        const NANOSECONDS: &str = "nanoseconds";
248        const TEST_SUITE: &str = "fuchsia.storage";
249        let results: Vec<FuchsiaPerfBenchmarkResult> = self
250            .results
251            .iter()
252            .map(|result| FuchsiaPerfBenchmarkResult {
253                label: format!("{}/{}", result.benchmark_name, result.filesystem_name),
254                test_suite: TEST_SUITE.to_owned(),
255                unit: NANOSECONDS.to_owned(),
256                values: result.values.iter().map(|d| d.0 as f64).collect(),
257            })
258            .collect();
259
260        serde_json::to_writer_pretty(writer, &results).unwrap()
261    }
262
263    /// Writes a summary of the benchmark results as a human readable table to `writer`.
264    pub fn write_table<F: Write>(&self, mut writer: F) {
265        let mut table = prettytable::Table::new();
266        table.set_format(*prettytable::format::consts::FORMAT_CLEAN);
267        table.add_row(BenchmarkResults::pretty_row_header());
268        for result in &self.results {
269            table.add_row(result.pretty_row());
270        }
271        table.print(&mut writer).unwrap();
272    }
273
274    /// Writes a summary of the benchmark results in csv format to `writer`.
275    pub fn write_csv<F: Write>(&self, mut writer: F) {
276        writer.write_all(BenchmarkResults::csv_row_header().as_bytes()).unwrap();
277        for result in &self.results {
278            writer.write_all(result.csv_row().as_bytes()).unwrap();
279        }
280    }
281}
282
283fn format_u64_with_commas(num: u64) -> String {
284    let mut result = String::new();
285    let num = num.to_string();
286    let digit_count = num.len();
287    for (i, d) in num.char_indices() {
288        result.push(d);
289        let rpos = digit_count - i;
290        if rpos != 1 && rpos % 3 == 1 {
291            result.push(',');
292        }
293    }
294    result
295}
296
297/// Macro to add many benchmark and filesystem pairs to a `BenchmarkSet`.
298///
299/// Expands:
300/// ```
301/// add_benchmarks!(benchmark_set, [benchmark1, benchmark2], [filesystem1, filesystem2]);
302/// ```
303/// Into:
304/// ```
305/// benchmark_set.add_benchmark(benchmark1.clone(), filesystem1.clone());
306/// benchmark_set.add_benchmark(benchmark1.clone(), filesystem2.clone());
307/// benchmark_set.add_benchmark(benchmark2.clone(), filesystem1.clone());
308/// benchmark_set.add_benchmark(benchmark2.clone(), filesystem2.clone());
309/// ```
310#[macro_export]
311macro_rules! add_benchmarks {
312    ($benchmark_set:ident, [$b:expr], [$f:expr]) => {
313        $benchmark_set.add_benchmark($b.clone(), $f.clone());
314    };
315    ($benchmark_set:ident, [$b:expr, $($bs:expr),+ $(,)?], [$($fs:expr),+ $(,)?]) => {
316        add_benchmarks!($benchmark_set, [$b], [$($fs),*]);
317        add_benchmarks!($benchmark_set, [$($bs),*], [$($fs),*]);
318    };
319    ($benchmark_set:ident, [$b:expr], [$f:expr, $($fs:expr),+ $(,)?]) => {
320        add_benchmarks!($benchmark_set, [$b], [$f]);
321        add_benchmarks!($benchmark_set, [$b], [$($fs),*]);
322    };
323}
324
325#[cfg(test)]
326mod tests {
327    use super::*;
328    use crate::block_device::PanickingBlockDeviceFactory;
329
330    #[derive(Clone)]
331    struct TestBenchmark(&'static str);
332
333    #[async_trait]
334    impl<T: Filesystem> Benchmark<T> for TestBenchmark {
335        async fn run(&self, _fs: &mut T) -> Vec<OperationDuration> {
336            vec![OperationDuration(100), OperationDuration(200), OperationDuration(300)]
337        }
338
339        fn name(&self) -> String {
340            self.0.to_owned()
341        }
342    }
343
344    #[derive(Clone)]
345    struct TestFilesystem(&'static str);
346
347    #[async_trait]
348    impl FilesystemConfig for TestFilesystem {
349        type Filesystem = TestFilesystemInstance;
350        async fn start_filesystem(
351            &self,
352            _block_device_factory: &dyn BlockDeviceFactory,
353        ) -> Self::Filesystem {
354            TestFilesystemInstance
355        }
356
357        fn name(&self) -> String {
358            self.0.to_string()
359        }
360    }
361
362    struct TestFilesystemInstance;
363
364    #[async_trait]
365    impl Filesystem for TestFilesystemInstance {
366        async fn shutdown(self) {}
367
368        fn benchmark_dir(&self) -> &std::path::Path {
369            panic!("not supported");
370        }
371    }
372
373    #[fuchsia::test]
374    async fn run_benchmark_set() {
375        let mut benchmark_set = BenchmarkSet::new();
376
377        let filesystem1 = TestFilesystem("filesystem1");
378        let filesystem2 = TestFilesystem("filesystem2");
379        let benchmark1 = TestBenchmark("benchmark1");
380        let benchmark2 = TestBenchmark("benchmark2");
381        benchmark_set.add_benchmark(benchmark1, filesystem1.clone());
382        benchmark_set.add_benchmark(benchmark2.clone(), filesystem1);
383        benchmark_set.add_benchmark(benchmark2, filesystem2);
384
385        let block_device_factory = PanickingBlockDeviceFactory::new();
386        let results = benchmark_set.run(&block_device_factory, &RegexSet::empty()).await;
387        let results = results.results;
388        assert_eq!(results.len(), 3);
389
390        assert_eq!(results[0].benchmark_name, "benchmark1");
391        assert_eq!(results[0].filesystem_name, "filesystem1");
392        assert_eq!(results[0].values.len(), 3);
393
394        assert_eq!(results[1].benchmark_name, "benchmark2");
395        assert_eq!(results[1].filesystem_name, "filesystem1");
396        assert_eq!(results[1].values.len(), 3);
397
398        assert_eq!(results[2].benchmark_name, "benchmark2");
399        assert_eq!(results[2].filesystem_name, "filesystem2");
400        assert_eq!(results[2].values.len(), 3);
401    }
402
403    #[fuchsia::test]
404    fn benchmark_filters() {
405        let config = BenchmarkConfigImpl {
406            benchmark: TestBenchmark("read_warm"),
407            filesystem_config: TestFilesystem("fs-name"),
408        };
409        // Accepted patterns.
410        assert!(config.matches(&RegexSet::empty()));
411        assert!(config.matches(&RegexSet::new([r""]).unwrap()));
412        assert!(config.matches(&RegexSet::new([r"fs-name"]).unwrap()));
413        assert!(config.matches(&RegexSet::new([r"/fs-name"]).unwrap()));
414        assert!(config.matches(&RegexSet::new([r"read_warm"]).unwrap()));
415        assert!(config.matches(&RegexSet::new([r"read_warm/"]).unwrap()));
416        assert!(config.matches(&RegexSet::new([r"read_warm/fs-name"]).unwrap()));
417        assert!(config.matches(&RegexSet::new([r"warm"]).unwrap()));
418
419        // Rejected patterns.
420        assert!(!config.matches(&RegexSet::new([r"cold"]).unwrap()));
421        assert!(!config.matches(&RegexSet::new([r"fxfs"]).unwrap()));
422        assert!(!config.matches(&RegexSet::new([r"^fs-name"]).unwrap()));
423        assert!(!config.matches(&RegexSet::new([r"read_warm$"]).unwrap()));
424
425        // Matches "warm".
426        assert!(config.matches(&RegexSet::new([r"warm", r"cold"]).unwrap()));
427        // Matches both.
428        assert!(config.matches(&RegexSet::new([r"warm", r"fs-name"]).unwrap()));
429        // Matches neither.
430        assert!(!config.matches(&RegexSet::new([r"cold", r"fxfs"]).unwrap()));
431    }
432
433    #[fuchsia::test]
434    fn result_statistics() {
435        let benchmark_name = "benchmark-name".to_string();
436        let filesystem_name = "filesystem-name".to_string();
437        let result = BenchmarkResults {
438            benchmark_name: benchmark_name.clone(),
439            filesystem_name: filesystem_name.clone(),
440            values: vec![
441                OperationDuration(600),
442                OperationDuration(700),
443                OperationDuration(500),
444                OperationDuration(800),
445                OperationDuration(400),
446            ],
447        };
448        let stats = result.statistics();
449        assert_eq!(stats.p50, 600);
450        assert_eq!(stats.p95, 780);
451        assert_eq!(stats.p99, 796);
452        assert_eq!(stats.mean, 600);
453        assert_eq!(stats.min, 400);
454        assert_eq!(stats.max, 800);
455        assert_eq!(stats.count, 5);
456    }
457
458    #[fuchsia::test]
459    fn percentile_test() {
460        let data = [400];
461        assert_eq!(percentile(&data, 50), 400);
462        assert_eq!(percentile(&data, 95), 400);
463        assert_eq!(percentile(&data, 99), 400);
464
465        let data = [400, 500];
466        assert_eq!(percentile(&data, 50), 450);
467        assert_eq!(percentile(&data, 95), 495);
468        assert_eq!(percentile(&data, 99), 499);
469
470        let data = [400, 500, 600];
471        assert_eq!(percentile(&data, 50), 500);
472        assert_eq!(percentile(&data, 95), 590);
473        assert_eq!(percentile(&data, 99), 598);
474
475        let data = [400, 500, 600, 700];
476        assert_eq!(percentile(&data, 50), 550);
477        assert_eq!(percentile(&data, 95), 685);
478        assert_eq!(percentile(&data, 99), 697);
479
480        let data = [400, 500, 600, 700, 800];
481        assert_eq!(percentile(&data, 50), 600);
482        assert_eq!(percentile(&data, 95), 780);
483        assert_eq!(percentile(&data, 99), 796);
484    }
485
486    #[fuchsia::test]
487    fn format_u64_with_commas_test() {
488        assert_eq!(format_u64_with_commas(0), "0");
489        assert_eq!(format_u64_with_commas(1), "1");
490        assert_eq!(format_u64_with_commas(99), "99");
491        assert_eq!(format_u64_with_commas(999), "999");
492        assert_eq!(format_u64_with_commas(1_000), "1,000");
493        assert_eq!(format_u64_with_commas(999_999), "999,999");
494        assert_eq!(format_u64_with_commas(1_000_000), "1,000,000");
495        assert_eq!(format_u64_with_commas(999_999_999), "999,999,999");
496        assert_eq!(format_u64_with_commas(1_000_000_000), "1,000,000,000");
497        assert_eq!(format_u64_with_commas(999_999_999_999), "999,999,999,999");
498    }
499
500    #[fuchsia::test]
501    fn add_benchmarks_test() {
502        let mut benchmark_set = BenchmarkSet::new();
503
504        let filesystem1 = TestFilesystem("filesystem1");
505        let filesystem2 = TestFilesystem("filesystem2");
506        let benchmark1 = TestBenchmark("benchmark1");
507        let benchmark2 = TestBenchmark("benchmark2");
508        add_benchmarks!(benchmark_set, [benchmark1], [filesystem1]);
509        add_benchmarks!(benchmark_set, [benchmark1], [filesystem1, filesystem2]);
510        add_benchmarks!(benchmark_set, [benchmark1, benchmark2], [filesystem1]);
511        add_benchmarks!(benchmark_set, [benchmark1, benchmark2], [filesystem1, filesystem2]);
512        add_benchmarks!(benchmark_set, [benchmark1, benchmark2,], [filesystem1, filesystem2,]);
513        let names: Vec<String> =
514            benchmark_set.benchmarks.iter().map(|config| config.name()).collect();
515        assert_eq!(
516            &names,
517            &[
518                "benchmark1/filesystem1",
519                "benchmark1/filesystem1",
520                "benchmark1/filesystem2",
521                "benchmark1/filesystem1",
522                "benchmark2/filesystem1",
523                "benchmark1/filesystem1",
524                "benchmark1/filesystem2",
525                "benchmark2/filesystem1",
526                "benchmark2/filesystem2",
527                "benchmark1/filesystem1",
528                "benchmark1/filesystem2",
529                "benchmark2/filesystem1",
530                "benchmark2/filesystem2"
531            ]
532        );
533    }
534}
storage_benchmarks/lib.rs

storage_benchmarks/
lib.rs
