1use crate::{Benchmark, CacheClearableFilesystem, Filesystem, OperationDuration, OperationTimer};
6use async_trait::async_trait;
7use std::collections::VecDeque;
8use std::ffi::{CStr, CString};
9use std::mem::MaybeUninit;
10use std::ops::Range;
11use std::os::fd::RawFd;
12use std::os::unix::ffi::OsStringExt;
13use std::path::{Path, PathBuf};
14
15#[derive(Copy, Clone)]
17pub struct DirectoryTreeStructure {
18 pub files_per_directory: u64,
20
21 pub directories_per_directory: u64,
24
25 pub max_depth: u32,
27}
28
29impl DirectoryTreeStructure {
30 fn max_pending(&self) -> u64 {
33 self.directories_per_directory.pow(self.max_depth)
34 }
35
36 fn create_directory_tree(&self, root: PathBuf) {
38 struct DirInfo {
39 path: PathBuf,
40 depth: u32,
41 }
42 let mut pending = VecDeque::new();
43 pending.push_back(DirInfo { path: root, depth: 0 });
44 while let Some(dir) = pending.pop_front() {
45 for i in 0..self.files_per_directory {
46 let path = dir.path.join(file_name(i));
47 std::fs::File::create(path).unwrap();
48 }
49 if self.max_depth == dir.depth {
50 continue;
51 }
52 for i in 0..self.directories_per_directory {
53 let path = dir.path.join(dir_name(i));
54 std::fs::create_dir(&path).unwrap();
55 pending.push_back(DirInfo { path, depth: dir.depth + 1 });
56 }
57 }
58 }
59
60 fn enumerate_paths(&self) -> Vec<PathBuf> {
62 let mut paths = Vec::new();
63 self.enumerate_paths_impl(Path::new(""), &mut paths);
64 paths
65 }
66
67 fn enumerate_paths_impl(&self, base: &Path, paths: &mut Vec<PathBuf>) {
68 for i in 0..self.files_per_directory {
69 paths.push(base.join(file_name(i)));
70 }
71 if self.max_depth == 0 {
72 return;
73 }
74 for i in 0..self.directories_per_directory {
75 let path = base.join(dir_name(i));
76 paths.push(path.clone());
77 Self { max_depth: self.max_depth - 1, ..*self }.enumerate_paths_impl(&path, paths);
78 }
79 }
80}
81
82#[derive(Clone)]
85pub struct WalkDirectoryTreeCold {
86 dts: DirectoryTreeStructure,
87 iterations: u64,
88}
89
90impl WalkDirectoryTreeCold {
91 pub fn new(dts: DirectoryTreeStructure, iterations: u64) -> Self {
92 Self { dts, iterations }
93 }
94}
95
96#[async_trait]
97impl<T: CacheClearableFilesystem> Benchmark<T> for WalkDirectoryTreeCold {
98 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
99 storage_trace::duration!(
100 c"benchmark",
101 c"WalkDirectoryTreeCold",
102 "files_per_directory" => self.dts.files_per_directory,
103 "directories_per_directory" => self.dts.directories_per_directory,
104 "max_depth" => self.dts.max_depth
105 );
106 let root = fs.benchmark_dir().to_path_buf();
107
108 self.dts.create_directory_tree(root.clone());
109 let max_pending = self.dts.max_pending() as usize;
110 let mut durations = Vec::new();
111 for i in 0..self.iterations {
112 fs.clear_cache().await;
113 storage_trace::duration!(c"benchmark", c"WalkDirectoryTree", "iteration" => i);
114 let timer = OperationTimer::start();
115 walk_directory_tree(root.clone(), max_pending);
116 durations.push(timer.stop());
117 }
118 durations
119 }
120
121 fn name(&self) -> String {
122 format!(
123 "WalkDirectoryTreeCold/Files{}/Dirs{}/Depth{}",
124 self.dts.files_per_directory, self.dts.directories_per_directory, self.dts.max_depth
125 )
126 }
127}
128
129#[derive(Clone)]
132pub struct WalkDirectoryTreeWarm {
133 dts: DirectoryTreeStructure,
134 iterations: u64,
135}
136
137impl WalkDirectoryTreeWarm {
138 pub fn new(dts: DirectoryTreeStructure, iterations: u64) -> Self {
139 Self { dts, iterations }
140 }
141}
142
143#[async_trait]
144impl<T: Filesystem> Benchmark<T> for WalkDirectoryTreeWarm {
145 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
146 storage_trace::duration!(
147 c"benchmark",
148 c"WalkDirectoryTreeWarm",
149 "files_per_directory" => self.dts.files_per_directory,
150 "directories_per_directory" => self.dts.directories_per_directory,
151 "max_depth" => self.dts.max_depth
152 );
153 let root = fs.benchmark_dir().to_path_buf();
154
155 self.dts.create_directory_tree(root.clone());
156 let max_pending = self.dts.max_pending() as usize;
157 let mut durations = Vec::new();
158 for i in 0..self.iterations {
159 storage_trace::duration!(c"benchmark", c"WalkDirectoryTree", "iteration" => i);
160 let timer = OperationTimer::start();
161 walk_directory_tree(root.clone(), max_pending);
162 durations.push(timer.stop());
163 }
164 durations
165 }
166
167 fn name(&self) -> String {
168 format!(
169 "WalkDirectoryTreeWarm/Files{}/Dirs{}/Depth{}",
170 self.dts.files_per_directory, self.dts.directories_per_directory, self.dts.max_depth
171 )
172 }
173}
174
175fn walk_directory_tree(root: PathBuf, max_pending: usize) {
178 let mut pending = VecDeque::new();
179 pending.reserve(max_pending);
180 pending.push_back(root);
181 while let Some(dir) = pending.pop_front() {
182 storage_trace::duration!(c"benchmark", c"read_dir");
183 for entry in std::fs::read_dir(&dir).unwrap() {
184 let entry = entry.unwrap();
185 if entry.file_type().unwrap().is_dir() {
186 pending.push_back(entry.path());
187 }
188 }
189 }
190}
191
192#[derive(Clone)]
195pub struct StatPath {
196 file_count: u64,
197}
198
199impl StatPath {
200 pub fn new() -> Self {
201 Self { file_count: 100 }
202 }
203}
204
205#[async_trait]
206impl<T: Filesystem> Benchmark<T> for StatPath {
207 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
208 storage_trace::duration!(c"benchmark", c"StatPath");
209
210 let root = fs.benchmark_dir().to_path_buf();
211 for i in 0..self.file_count {
212 std::fs::File::create(root.join(file_name(i))).unwrap();
213 }
214
215 let root_fd =
216 open_path(&path_buf_to_c_string(root), libc::O_DIRECTORY | libc::O_RDONLY).unwrap();
217
218 let mut durations = Vec::with_capacity(self.file_count as usize);
219 for i in 0..self.file_count {
220 let path = path_buf_to_c_string(file_name(i));
221 storage_trace::duration!(c"benchmark", c"stat", "file" => i);
222 let timer = OperationTimer::start();
223 stat_path_at(&root_fd, &path).unwrap();
224 durations.push(timer.stop());
225 }
226 durations
227 }
228
229 fn name(&self) -> String {
230 "StatPath".to_string()
231 }
232}
233
234#[derive(Clone)]
237pub struct OpenFile {
238 file_count: u64,
239}
240
241impl OpenFile {
242 pub fn new() -> Self {
243 Self { file_count: 100 }
244 }
245}
246
247#[async_trait]
248impl<T: Filesystem> Benchmark<T> for OpenFile {
249 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
250 storage_trace::duration!(c"benchmark", c"OpenFile");
251
252 let root = fs.benchmark_dir().to_path_buf();
253 for i in 0..self.file_count {
254 std::fs::File::create(root.join(file_name(i))).unwrap();
255 }
256
257 let root_fd =
258 open_path(&path_buf_to_c_string(root), libc::O_DIRECTORY | libc::O_RDONLY).unwrap();
259
260 let mut durations = Vec::with_capacity(self.file_count as usize);
261 for i in 0..self.file_count {
262 let path = path_buf_to_c_string(file_name(i));
263 let _file = {
265 storage_trace::duration!(c"benchmark", c"open", "file" => i);
266 let timer = OperationTimer::start();
267 let file = open_path_at(&root_fd, &path, libc::O_RDWR);
268 durations.push(timer.stop());
269 file
270 };
271 }
272 durations
273 }
274
275 fn name(&self) -> String {
276 "OpenFile".to_string()
277 }
278}
279
280#[derive(Clone)]
282pub struct CreateFile {
283 file_count: u64,
284}
285
286impl CreateFile {
287 pub fn new() -> Self {
288 Self { file_count: 100 }
289 }
290}
291
292#[async_trait]
293impl<T: Filesystem> Benchmark<T> for CreateFile {
294 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
295 storage_trace::duration!(c"benchmark", c"CreateFile");
296
297 let root = fs.benchmark_dir().to_path_buf();
298 let root_fd =
299 open_path(&path_buf_to_c_string(root), libc::O_DIRECTORY | libc::O_RDONLY).unwrap();
300
301 let mut durations = Vec::with_capacity(self.file_count as usize);
302 for i in 0..self.file_count {
303 let path = path_buf_to_c_string(file_name(i));
304 let _file = {
306 storage_trace::duration!(c"benchmark", c"create", "file" => i);
307 let timer = OperationTimer::start();
308 let file = open_path_at(&root_fd, &path, libc::O_CREAT | libc::O_RDWR);
309 durations.push(timer.stop());
310 file
311 };
312 }
313 durations
314 }
315
316 fn name(&self) -> String {
317 "CreateFile".to_string()
318 }
319}
320
321#[derive(Clone)]
324pub struct OpenDeeplyNestedFile {
325 file_count: u64,
326 depth: u64,
327}
328
329impl OpenDeeplyNestedFile {
330 pub fn new() -> Self {
331 Self { file_count: 100, depth: 5 }
332 }
333
334 fn dir_path(&self, n: u64) -> PathBuf {
335 let mut path = PathBuf::new();
336 for i in 0..self.depth {
337 path = path.join(format!("dir-{:03}-{:03}", n, i));
338 }
339 path
340 }
341}
342
343#[async_trait]
344impl<T: Filesystem> Benchmark<T> for OpenDeeplyNestedFile {
345 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
346 storage_trace::duration!(c"benchmark", c"OpenDeeplyNestedFile");
347
348 let root = fs.benchmark_dir().to_path_buf();
349 for i in 0..self.file_count {
350 let dir_path = root.join(self.dir_path(i));
351 std::fs::create_dir_all(&dir_path).unwrap();
352 std::fs::File::create(dir_path.join(file_name(i))).unwrap();
353 }
354
355 let root_fd =
356 open_path(&path_buf_to_c_string(root), libc::O_DIRECTORY | libc::O_RDONLY).unwrap();
357
358 let mut durations = Vec::with_capacity(self.file_count as usize);
359 for i in 0..self.file_count {
360 let path = path_buf_to_c_string(self.dir_path(i).join(file_name(i)));
361 let _file = {
363 storage_trace::duration!(c"benchmark", c"open", "file" => i);
364 let timer = OperationTimer::start();
365 let file = open_path_at(&root_fd, &path, libc::O_RDWR);
366 durations.push(timer.stop());
367 file
368 };
369 }
370 durations
371 }
372
373 fn name(&self) -> String {
374 "OpenDeeplyNestedFile".to_string()
375 }
376}
377
378#[derive(Clone)]
380pub struct GitStatus {
381 dts: DirectoryTreeStructure,
382 iterations: u64,
383
384 stat_threads: usize,
386}
387
388impl GitStatus {
389 pub fn new() -> Self {
390 let dts = DirectoryTreeStructure {
393 files_per_directory: 4,
394 directories_per_directory: 2,
395 max_depth: 7,
396 };
397
398 Self { dts, iterations: 5, stat_threads: 1 }
401 }
402
403 fn stat_paths(&self, root: &OpenFd, paths: &Vec<CString>) {
406 storage_trace::duration!(c"benchmark", c"GitStatus::stat_paths");
407 std::thread::scope(|scope| {
408 for thread in 0..self.stat_threads {
409 scope.spawn(move || {
410 let paths = &paths;
411 for path in batch_range(paths.len(), self.stat_threads, thread) {
412 storage_trace::duration!(c"benchmark", c"stat");
413 stat_path_at(root, &paths[path]).unwrap();
414 }
415 });
416 }
417 });
418 }
419
420 fn walk_repo(&self, dir: &Path) {
422 storage_trace::duration!(c"benchmark", c"GitStatus::walk_repo");
423 for entry in std::fs::read_dir(&dir).unwrap() {
424 let entry = entry.unwrap();
425 if entry.file_type().unwrap().is_dir() {
426 self.walk_repo(&entry.path());
427 }
428 }
429 }
430}
431
432#[async_trait]
433impl<T: Filesystem> Benchmark<T> for GitStatus {
434 async fn run(&self, fs: &mut T) -> Vec<OperationDuration> {
435 let root = &fs.benchmark_dir().to_path_buf();
436
437 self.dts.create_directory_tree(root.clone());
438 let paths = self.dts.enumerate_paths().into_iter().map(path_buf_to_c_string).collect();
439
440 let root_fd =
441 open_path(&path_buf_to_c_string(root.clone()), libc::O_DIRECTORY | libc::O_RDONLY)
442 .unwrap();
443
444 let mut durations = Vec::new();
445 for i in 0..self.iterations {
446 storage_trace::duration!(c"benchmark", c"GitStatus", "iteration" => i);
447 let timer = OperationTimer::start();
448 self.stat_paths(&root_fd, &paths);
449 self.walk_repo(&root);
450 durations.push(timer.stop());
451 }
452 durations
453 }
454
455 fn name(&self) -> String {
456 "GitStatus".to_owned()
457 }
458}
459
460pub fn file_name(n: u64) -> PathBuf {
461 format!("file-{:03}.txt", n).into()
462}
463
464pub fn dir_name(n: u64) -> PathBuf {
465 format!("dir-{:03}", n).into()
466}
467
468pub fn path_buf_to_c_string(path: PathBuf) -> CString {
469 CString::new(path.into_os_string().into_vec()).unwrap()
470}
471
472pub fn stat_path_at(dir: &OpenFd, path: &CStr) -> Result<libc::stat, std::io::Error> {
473 unsafe {
474 let mut stat = MaybeUninit::uninit();
475 let result =
478 libc::fstatat(dir.0, path.as_ptr(), stat.as_mut_ptr(), libc::AT_SYMLINK_NOFOLLOW);
479
480 if result == 0 { Ok(stat.assume_init()) } else { Err(std::io::Error::last_os_error()) }
481 }
482}
483
484fn batch_range(item_count: usize, batch_count: usize, batch_num: usize) -> Range<usize> {
487 let items_per_batch = (item_count + (batch_count - 1)) / batch_count;
488 let start = items_per_batch * batch_num;
489 let end = std::cmp::min(item_count, start + items_per_batch);
490 start..end
491}
492
493pub struct OpenFd(RawFd);
494
495impl Drop for OpenFd {
496 fn drop(&mut self) {
497 unsafe { libc::close(self.0) };
498 }
499}
500
501pub fn open_path(path: &CStr, flags: libc::c_int) -> Result<OpenFd, std::io::Error> {
502 let result = unsafe { libc::open(path.as_ptr(), flags) };
503 if result >= 0 { Ok(OpenFd(result)) } else { Err(std::io::Error::last_os_error()) }
504}
505
506pub fn open_path_at(
507 dir: &OpenFd,
508 path: &CStr,
509 flags: libc::c_int,
510) -> Result<OpenFd, std::io::Error> {
511 let result = unsafe { libc::openat(dir.0, path.as_ptr(), flags) };
512 if result >= 0 { Ok(OpenFd(result)) } else { Err(std::io::Error::last_os_error()) }
513}
514
515#[cfg(test)]
516mod tests {
517 use super::*;
518 use crate::testing::TestFilesystem;
519
520 const ITERATION_COUNT: u64 = 3;
521
522 #[derive(PartialEq, Debug)]
523 struct DirectoryTree {
524 files: u64,
525 directories: Vec<DirectoryTree>,
526 }
527
528 fn read_in_directory_tree(root: PathBuf) -> DirectoryTree {
529 let mut tree = DirectoryTree { files: 0, directories: vec![] };
530 for entry in std::fs::read_dir(root).unwrap() {
531 let entry = entry.unwrap();
532 if entry.file_type().unwrap().is_dir() {
533 tree.directories.push(read_in_directory_tree(entry.path()));
534 } else {
535 tree.files += 1;
536 }
537 }
538 tree
539 }
540
541 #[fuchsia::test]
542 async fn create_directory_tree_with_depth_of_zero_test() {
543 let test_fs = Box::new(TestFilesystem::new());
544 let dts = DirectoryTreeStructure {
545 files_per_directory: 3,
546 directories_per_directory: 2,
547 max_depth: 0,
548 };
549 let root = test_fs.benchmark_dir().to_owned();
550 dts.create_directory_tree(root.clone());
551
552 let directory_tree = read_in_directory_tree(root);
553 assert_eq!(directory_tree, DirectoryTree { files: 3, directories: vec![] });
554 test_fs.shutdown().await;
555 }
556
557 #[fuchsia::test]
558 async fn create_directory_tree_with_depth_of_two_test() {
559 let test_fs = Box::new(TestFilesystem::new());
560 let dts = DirectoryTreeStructure {
561 files_per_directory: 4,
562 directories_per_directory: 2,
563 max_depth: 2,
564 };
565 let root = test_fs.benchmark_dir().to_owned();
566 dts.create_directory_tree(root.clone());
567
568 let directory_tree = read_in_directory_tree(root);
569 assert_eq!(
570 directory_tree,
571 DirectoryTree {
572 files: 4,
573 directories: vec![
574 DirectoryTree {
575 files: 4,
576 directories: vec![
577 DirectoryTree { files: 4, directories: vec![] },
578 DirectoryTree { files: 4, directories: vec![] }
579 ]
580 },
581 DirectoryTree {
582 files: 4,
583 directories: vec![
584 DirectoryTree { files: 4, directories: vec![] },
585 DirectoryTree { files: 4, directories: vec![] }
586 ]
587 }
588 ]
589 }
590 );
591 test_fs.shutdown().await;
592 }
593
594 #[fuchsia::test]
595 fn enumerate_paths_test() {
596 let dts = DirectoryTreeStructure {
597 files_per_directory: 2,
598 directories_per_directory: 2,
599 max_depth: 0,
600 };
601 let paths = dts.enumerate_paths();
602 assert_eq!(paths, vec![PathBuf::from("file-000.txt"), PathBuf::from("file-001.txt")],);
603
604 let dts = DirectoryTreeStructure {
605 files_per_directory: 2,
606 directories_per_directory: 2,
607 max_depth: 1,
608 };
609 let paths = dts.enumerate_paths();
610 assert_eq!(
611 paths,
612 vec![
613 PathBuf::from("file-000.txt"),
614 PathBuf::from("file-001.txt"),
615 PathBuf::from("dir-000"),
616 PathBuf::from("dir-000/file-000.txt"),
617 PathBuf::from("dir-000/file-001.txt"),
618 PathBuf::from("dir-001"),
619 PathBuf::from("dir-001/file-000.txt"),
620 PathBuf::from("dir-001/file-001.txt"),
621 ],
622 );
623 }
624
625 #[fuchsia::test]
626 fn batch_range_test() {
627 assert_eq!(batch_range(10, 1, 0), 0..10);
628
629 assert_eq!(batch_range(10, 3, 0), 0..4);
630 assert_eq!(batch_range(10, 3, 1), 4..8);
631 assert_eq!(batch_range(10, 3, 2), 8..10);
632
633 assert_eq!(batch_range(10, 4, 3), 9..10);
634 assert_eq!(batch_range(12, 4, 3), 9..12);
635 }
636
637 #[fuchsia::test]
638 async fn walk_directory_tree_cold_test() {
639 let mut test_fs = Box::new(TestFilesystem::new());
640 let dts = DirectoryTreeStructure {
641 files_per_directory: 2,
642 directories_per_directory: 2,
643 max_depth: 2,
644 };
645 let results = WalkDirectoryTreeCold::new(dts, ITERATION_COUNT).run(test_fs.as_mut()).await;
646
647 assert_eq!(results.len(), ITERATION_COUNT as usize);
648 assert_eq!(test_fs.clear_cache_count().await, ITERATION_COUNT);
649 test_fs.shutdown().await;
650 }
651
652 #[fuchsia::test]
653 async fn walk_directory_tree_warm_test() {
654 let mut test_fs = Box::new(TestFilesystem::new());
655 let dts = DirectoryTreeStructure {
656 files_per_directory: 2,
657 directories_per_directory: 2,
658 max_depth: 2,
659 };
660 let results = WalkDirectoryTreeWarm::new(dts, ITERATION_COUNT).run(test_fs.as_mut()).await;
661
662 assert_eq!(results.len(), ITERATION_COUNT as usize);
663 assert_eq!(test_fs.clear_cache_count().await, 0);
664 test_fs.shutdown().await;
665 }
666
667 #[fuchsia::test]
668 async fn stat_path_test() {
669 let mut test_fs = Box::new(TestFilesystem::new());
670 let benchmark = StatPath { file_count: 5 };
671 let results = benchmark.run(test_fs.as_mut()).await;
672 assert_eq!(results.len(), 5);
673 assert_eq!(test_fs.clear_cache_count().await, 0);
674 test_fs.shutdown().await;
675 }
676
677 #[fuchsia::test]
678 async fn open_file_test() {
679 let mut test_fs = Box::new(TestFilesystem::new());
680 let benchmark = OpenFile { file_count: 5 };
681 let results = benchmark.run(test_fs.as_mut()).await;
682 assert_eq!(results.len(), 5);
683 assert_eq!(test_fs.clear_cache_count().await, 0);
684 test_fs.shutdown().await;
685 }
686
687 #[fuchsia::test]
688 async fn create_file_test() {
689 let mut test_fs = Box::new(TestFilesystem::new());
690 let benchmark = CreateFile { file_count: 5 };
691 let results = benchmark.run(test_fs.as_mut()).await;
692 assert_eq!(results.len(), 5);
693 assert_eq!(test_fs.clear_cache_count().await, 0);
694 test_fs.shutdown().await;
695 }
696
697 #[fuchsia::test]
698 async fn open_deeply_nested_file_test() {
699 let mut test_fs = Box::new(TestFilesystem::new());
700 let benchmark = OpenDeeplyNestedFile { file_count: 5, depth: 3 };
701 let results = benchmark.run(test_fs.as_mut()).await;
702 assert_eq!(results.len(), 5);
703 assert_eq!(test_fs.clear_cache_count().await, 0);
704 test_fs.shutdown().await;
705 }
706
707 #[fuchsia::test]
708 async fn git_status_test() {
709 let mut test_fs = Box::new(TestFilesystem::new());
710 let dts = DirectoryTreeStructure {
711 files_per_directory: 2,
712 directories_per_directory: 2,
713 max_depth: 2,
714 };
715 let benchmark = GitStatus { dts, iterations: ITERATION_COUNT, stat_threads: 1 };
716 let results = benchmark.run(test_fs.as_mut()).await;
717
718 assert_eq!(results.len(), ITERATION_COUNT as usize);
719 assert_eq!(test_fs.clear_cache_count().await, 0);
720 test_fs.shutdown().await;
721 }
722}