test_output_directory/
testing.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::{
6    ArtifactMetadata, ArtifactType, MaybeUnknown, Outcome, SuiteResult, TestCaseResult,
7    TestRunResult,
8};
9use std::collections::{HashMap, HashSet};
10use std::ops::Deref;
11use std::path::{Path, PathBuf};
12use test_list::TestTag;
13
14enum MatchOption<T> {
15    AnyOrNone,
16    None,
17    Any,
18    Specified(T),
19}
20
21macro_rules! assert_match_option {
22    ($expected:expr, $actual:expr, $field:expr) => {
23        match $expected {
24            MatchOption::AnyOrNone => (),
25            MatchOption::None => {
26                assert_eq!(None, $actual, "Expected {} to be None but was {:?}", $field, $actual)
27            }
28            MatchOption::Any => {
29                assert!($actual.is_some(), "Expected {} to contain a value but was None", $field)
30            }
31            MatchOption::Specified(val) => assert_eq!(
32                Some(val),
33                $actual,
34                "Expected {} to be {:?} but was {:?}",
35                $field,
36                Some(val),
37                $actual
38            ),
39        }
40    };
41}
42
43/// Container that identifies the entity that is being verified in an assertion.
44#[derive(Clone, Copy)]
45enum EntityContext<'a> {
46    Run,
47    Suite(&'a ExpectedSuite),
48    Case(&'a ExpectedSuite, &'a ExpectedTestCase),
49}
50
51/// Container that identifies the artifact that is being verified in an assertion.
52#[derive(Clone, Copy)]
53struct ArtifactContext<'a, 'b> {
54    entity: &'a EntityContext<'b>,
55    metadata: &'a ArtifactMetadata,
56}
57
58impl std::fmt::Display for EntityContext<'_> {
59    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
60        match self {
61            Self::Run => write!(f, "TEST RUN"),
62            Self::Suite(suite) => write!(f, "SUITE {}", suite.name),
63            Self::Case(suite, case) => write!(f, "SUITE {}: CASE {}", suite.name, case.name),
64        }
65    }
66}
67
68impl std::fmt::Display for ArtifactContext<'_, '_> {
69    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
70        write!(f, "Entity: {}, Metadata: {:?}", self.entity, self.metadata)
71    }
72}
73
74/// A mapping from artifact metadata to assertions made on the artifact.
75type ArtifactMetadataToAssertionMap = HashMap<ArtifactMetadata, ExpectedArtifact>;
76
77/// Assert that the run results contained in `actual_run` and the directory specified by `root`
78/// contain the results and artifacts in `expected_run`.
79pub fn assert_run_result(root: &Path, expected_run: &ExpectedTestRun) {
80    let context = EntityContext::Run;
81    let actual_run = TestRunResult::from_dir(root).expect("Parse output directory");
82    let TestRunResult { common, suites } = actual_run;
83    assert_match_option!(
84        expected_run.duration_milliseconds,
85        common.deref().duration_milliseconds,
86        format!("Run duration for {}", context)
87    );
88    assert_match_option!(
89        expected_run.start_time,
90        common.deref().start_time,
91        format!("Start time for {}", context)
92    );
93    assert_eq!(common.deref().outcome, expected_run.outcome, "Outcome for {}", context);
94    assert_artifacts(
95        root,
96        &common.deref().artifact_dir.root,
97        &common.deref().artifact_dir.artifacts,
98        &expected_run.artifacts,
99        EntityContext::Run,
100    );
101    assert_suite_results(root, &suites, &expected_run.suites);
102}
103
104/// Assert that the suite results contained in `actual_suites` and the directory specified by `root`
105/// contain the suites, results, artifacts, and test cases in `expected_suite`.
106/// Note that this currently does not support duplicate suite names.
107fn assert_suite_results(
108    root: &Path,
109    actual_suites: &Vec<SuiteResult<'_>>,
110    expected_suites: &Vec<ExpectedSuite>,
111) {
112    assert_eq!(actual_suites.len(), expected_suites.len());
113    let mut expected_suites_map = HashMap::new();
114    for suite in expected_suites.iter() {
115        expected_suites_map.insert(suite.name.clone(), suite);
116    }
117    assert_eq!(
118        actual_suites.len(),
119        expected_suites_map.len(),
120        "Run contains multiple suites with the same name. \
121        This is currently unsupported by assert_suite_results"
122    );
123    for suite in actual_suites.iter() {
124        assert_suite_result(
125            root,
126            suite,
127            expected_suites_map
128                .get(&suite.common.deref().name)
129                .expect("No matching expected suite"),
130        );
131    }
132}
133
134/// Assert that the suite results contained in `actual_suite` and the directory specified by `root`
135/// contain the results, artifacts, and test cases in `expected_suite`.
136pub fn assert_suite_result(
137    root: &Path,
138    actual_suite: &SuiteResult<'_>,
139    expected_suite: &ExpectedSuite,
140) {
141    let context = EntityContext::Suite(expected_suite);
142    let &SuiteResult { common, cases, tags } = &actual_suite;
143    assert_eq!(common.deref().outcome, expected_suite.outcome, "Outcome for {}", context);
144    assert_eq!(common.deref().name, expected_suite.name, "Name for {}", context);
145    assert_match_option!(
146        expected_suite.duration_milliseconds,
147        common.deref().duration_milliseconds,
148        format!("Duration for {}", context)
149    );
150    assert_match_option!(
151        expected_suite.start_time,
152        common.deref().start_time,
153        format!("Start time for {}", context)
154    );
155
156    let mut tags: Vec<TestTag> = tags.clone().into_owned();
157    tags.sort();
158
159    let mut expected_tags = expected_suite.tags.clone();
160    expected_tags.sort();
161
162    assert_eq!(tags, expected_tags);
163
164    // Filter out the debug artifacts. Debug artifacts can be generated or not by the same
165    // test. This optional presence can't be expressed gy the Expected types. This wasn't a
166    // problem when the debug artifact was associated with the run (rather than the suite),
167    // because we simply refrain from checking artifacts on the run. We do check artifacts
168    // on the suite, so we remove the debug artifact here. This turns out to be much easier
169    // than adding optionality to artifact expectations.
170    let filtered_artifacts: HashMap<PathBuf, ArtifactMetadata> = common
171        .deref()
172        .artifact_dir
173        .artifacts
174        .clone()
175        .into_iter()
176        .filter(|(_, value)| value.artifact_type != ArtifactType::Debug.into())
177        .collect();
178
179    assert_artifacts(
180        root,
181        &common.deref().artifact_dir.root,
182        &filtered_artifacts,
183        &expected_suite.artifacts,
184        context,
185    );
186
187    assert_eq!(cases.len(), expected_suite.cases.len());
188    for case in cases.iter() {
189        let expected_case = expected_suite.cases.get(&case.common.deref().name);
190        assert!(
191            expected_case.is_some(),
192            "Found unexpected case {} in {}",
193            case.common.deref().name,
194            context
195        );
196        assert_case_result(root, case, expected_case.unwrap(), expected_suite);
197    }
198}
199
200fn assert_case_result(
201    root: &Path,
202    actual_case: &TestCaseResult<'_>,
203    expected_case: &ExpectedTestCase,
204    parent_suite: &ExpectedSuite,
205) {
206    let context = EntityContext::Case(parent_suite, expected_case);
207    assert_eq!(actual_case.common.deref().name, expected_case.name, "Name for {}", context);
208    assert_eq!(
209        actual_case.common.deref().outcome,
210        expected_case.outcome,
211        "Outcome for {}",
212        context
213    );
214    assert_match_option!(
215        expected_case.duration_milliseconds,
216        actual_case.common.deref().duration_milliseconds,
217        format!("Duration for {}", context)
218    );
219    assert_match_option!(
220        expected_case.start_time,
221        actual_case.common.deref().start_time,
222        format!("Start time for {}", context)
223    );
224    assert_artifacts(
225        root,
226        &actual_case.common.deref().artifact_dir.root,
227        &actual_case.common.deref().artifact_dir.artifacts,
228        &expected_case.artifacts,
229        context,
230    );
231}
232
233fn assert_artifacts(
234    root: &Path,
235    artifact_dir: &Path,
236    actual_artifacts: &HashMap<PathBuf, ArtifactMetadata>,
237    expected_artifacts: &ArtifactMetadataToAssertionMap,
238    entity_context: EntityContext<'_>,
239) {
240    // TODO(https://fxbug.dev/42051180): add options so that the test author can explicitly declare whether
241    // artifacts should be an exact match, should contain (and may contain more) artifacts,
242    // or any number of artifacts is accesptable.
243    // This skips artifact assertion for the typical case where verifying artifacts isn't
244    // necessary and allows the author to avoid listing out every artifact that is generated
245    // by the test.
246    if expected_artifacts.is_empty() {
247        return;
248    }
249
250    let actual_artifacts_by_metadata: HashMap<ArtifactMetadata, PathBuf> =
251        actual_artifacts.iter().map(|(key, value)| (value.clone(), key.clone())).collect();
252    // For now, artifact metadata should be unique for each artifact.
253    assert_eq!(
254        actual_artifacts_by_metadata.len(),
255        actual_artifacts.len(),
256        "Artifacts for {} do not have unique metadata. Actual artifacts: {:?}",
257        entity_context,
258        actual_artifacts
259    );
260
261    let expected_metadata: HashSet<_> = expected_artifacts.keys().collect();
262    let actual_metadata: HashSet<_> = actual_artifacts_by_metadata.keys().collect();
263
264    assert_eq!(
265        expected_metadata, actual_metadata,
266        "Artifacts for {} do not have matching metadata.",
267        entity_context,
268    );
269
270    for (expected_metadata, expected_artifact) in expected_artifacts.iter() {
271        let actual_filepath =
272            artifact_dir.join(actual_artifacts_by_metadata.get(expected_metadata).unwrap());
273        match expected_artifact {
274            ExpectedArtifact::File { name, assertion_fn } => {
275                assert_file(
276                    &root.join(&actual_filepath),
277                    name,
278                    assertion_fn,
279                    ArtifactContext { entity: &entity_context, metadata: expected_metadata },
280                );
281            }
282            ExpectedArtifact::Directory { files, name } => {
283                match name {
284                    None => (),
285                    Some(name) => assert_eq!(
286                        name.as_str(),
287                        actual_filepath.file_name().unwrap().to_str().unwrap(),
288                        "Expected filename {} for artifact matching {:?} but got {}",
289                        name,
290                        expected_metadata,
291                        actual_filepath.file_name().unwrap().to_str().unwrap()
292                    ),
293                }
294                let actual_entries: HashSet<_> = std::fs::read_dir(root.join(&actual_filepath))
295                    .expect("Failed to read directory artifact path")
296                    .map(|entry| match entry {
297                        Ok(dir_entry) if dir_entry.file_type().unwrap().is_file() => {
298                            dir_entry.file_name().to_str().unwrap().to_string()
299                        }
300                        // TODO(fxbugdev/85528) - support directory artifacts with subdirectories
301                        Ok(_) => panic!("Directory artifact with subdirectories unsupported"),
302                        Err(e) => panic!("Error reading directory artifact: {:?}", e),
303                    })
304                    .collect();
305                let expected_entries: HashSet<_> =
306                    files.iter().map(|(name, _)| name.to_string()).collect();
307                assert_eq!(
308                    actual_entries, expected_entries,
309                    "Expected files {:?} in directory artifact, got {:?}",
310                    &expected_entries, &actual_entries
311                );
312                for (name, assertion) in files {
313                    assert_file(
314                        &root.join(&actual_filepath).join(name),
315                        &None,
316                        assertion,
317                        ArtifactContext { entity: &entity_context, metadata: expected_metadata },
318                    );
319                }
320            }
321        }
322    }
323}
324
325fn assert_file(
326    file_path: &Path,
327    name: &Option<String>,
328    assertion_fn: &Box<dyn Fn(&str)>,
329    artifact_context: ArtifactContext<'_, '_>,
330) {
331    match name {
332        None => (),
333        Some(name) => assert_eq!(
334            name.as_str(),
335            file_path.file_name().unwrap().to_str().unwrap(),
336            "Got incorrect filename while checking file for artifact {}",
337            artifact_context
338        ),
339    }
340    let actual_contents = std::fs::read_to_string(&file_path);
341    (assertion_fn)(&actual_contents.unwrap());
342}
343
344/// The expected contents of an artifact.
345enum ExpectedArtifact {
346    /// An artifact contained in a single file, such as stdout.
347    File {
348        /// If given, the expected name of the file.
349        name: Option<String>,
350        /// Assertion run against the contents of the file.
351        assertion_fn: Box<dyn Fn(&str)>,
352    },
353    /// An artifact consisting of files in a directory.
354    Directory {
355        /// List of expected files, as (name, assertion) pairs. The name
356        /// is the expected name of the file, and the assertion fn is run
357        /// against the contents of the file.
358        files: Vec<(String, Box<dyn Fn(&str)>)>,
359        /// If given, the expected name of the directory.
360        name: Option<String>,
361    },
362}
363
364/// Contents of an expected directory artifact.
365pub struct ExpectedDirectory {
366    files: Vec<(String, Box<dyn Fn(&str)>)>,
367}
368
369impl ExpectedDirectory {
370    /// Create a new empty expected directory.
371    pub fn new() -> Self {
372        Self { files: vec![] }
373    }
374
375    /// Add a file with expected |contents|.
376    pub fn with_file(self, name: impl AsRef<str>, contents: impl AsRef<str>) -> Self {
377        let owned_expected = contents.as_ref().to_string();
378        let owned_name = name.as_ref().to_string();
379        self.with_matching_file(name, move |actual| {
380            assert_eq!(
381                &owned_expected, actual,
382                "Mismatch in contents of file {}. Expected: '{}', actual:'{}'",
383                owned_name, &owned_expected, actual
384            )
385        })
386    }
387
388    pub fn with_matching_file(
389        mut self,
390        name: impl AsRef<str>,
391        matcher: impl 'static + Fn(&str),
392    ) -> Self {
393        self.files.push((name.as_ref().to_string(), Box::new(matcher)));
394        self
395    }
396}
397
398/// A version of a test run result that contains all output in memory. This should only be used
399/// for making assertions in a test.
400pub struct ExpectedTestRun {
401    artifacts: ArtifactMetadataToAssertionMap,
402    outcome: MaybeUnknown<Outcome>,
403    start_time: MatchOption<u64>,
404    duration_milliseconds: MatchOption<u64>,
405    suites: Vec<ExpectedSuite>,
406}
407
408/// A version of a suite run result that contains all output in memory. This should only be used
409/// for making assertions in a test.
410pub struct ExpectedSuite {
411    artifacts: ArtifactMetadataToAssertionMap,
412    name: String,
413    outcome: MaybeUnknown<Outcome>,
414    cases: HashMap<String, ExpectedTestCase>,
415    start_time: MatchOption<u64>,
416    duration_milliseconds: MatchOption<u64>,
417    tags: Vec<TestTag>,
418}
419
420/// A version of a test case result that contains all output in memory. This should only be used
421/// for making assertions in a test.
422pub struct ExpectedTestCase {
423    artifacts: ArtifactMetadataToAssertionMap,
424    name: String,
425    outcome: MaybeUnknown<Outcome>,
426    start_time: MatchOption<u64>,
427    duration_milliseconds: MatchOption<u64>,
428}
429
430macro_rules! common_impl {
431    {} => {
432        /// Add an artifact matching the exact contents. Artifacts are checked by finding
433        /// an entry matching the given metadata, then checking the contents of the corresponding
434        /// file. If |name| is provided, the name of the file is verified. Artifacts are keyed by
435        /// metadata rather than by name as the names of files are not guaranteed to be stable.
436        pub fn with_artifact<S, T, U>(
437            self, metadata: U, name: Option<S>, contents: T
438        ) -> Self
439        where
440            S: AsRef<str>,
441            T: AsRef<str>,
442            U: Into<ArtifactMetadata>
443        {
444            let owned_expected = contents.as_ref().to_string();
445            let metadata = metadata.into();
446            let metadata_clone = metadata.clone();
447            self.with_matching_artifact(metadata, name, move |actual| {
448                assert_eq!(
449                    &owned_expected, actual,
450                    "Mismatch in artifact with metadata {:?}. Expected: '{}', actual:'{}'",
451                    metadata_clone, &owned_expected, actual
452                )
453            })
454        }
455
456        /// Add an artifact matching the exact contents. Artifacts are checked by finding
457        /// an entry matching the given metadata, then running |matcher| against the contents of
458        /// the file. If |name| is provided, the name of the file is verified. Artifacts are keyed
459        /// by metadata rather than by name as the names of files are not guaranteed to be stable.
460        pub fn with_matching_artifact<S, F, U>(
461            mut self,
462            metadata: U,
463            name: Option<S>,
464            matcher: F,
465        ) -> Self
466        where
467            S: AsRef<str>,
468            F: 'static + Fn(&str),
469            U: Into<ArtifactMetadata>
470        {
471            self.artifacts.insert(
472                metadata.into(),
473                ExpectedArtifact::File {
474                    name: name.map(|s| s.as_ref().to_string()),
475                    assertion_fn: Box::new(matcher),
476                }
477            );
478            self
479        }
480
481        /// Add a directory based artifact containing the entries described in |directory|.
482        pub fn with_directory_artifact<S, U>(
483            mut self,
484            metadata: U,
485            name: Option<S>,
486            directory: ExpectedDirectory,
487        ) -> Self
488        where
489            S: AsRef<str>,
490            U: Into<ArtifactMetadata>
491        {
492            self.artifacts.insert(
493                metadata.into(),
494                ExpectedArtifact::Directory {
495                    name: name.map(|s| s.as_ref().to_string()),
496                    files: directory.files,
497                }
498            );
499            self
500        }
501
502        /// Verify an exact start time.
503        pub fn with_start_time(mut self, millis: u64) -> Self {
504            self.start_time = MatchOption::Specified(millis);
505            self
506        }
507
508        /// Verify an exact run duration.
509        pub fn with_run_duration(mut self, millis: u64) -> Self {
510            self.duration_milliseconds = MatchOption::Specified(millis);
511            self
512        }
513
514        /// Verify that a start time is present.
515        pub fn with_any_start_time(mut self) -> Self {
516            self.start_time = MatchOption::Any;
517            self
518        }
519
520        /// Verify that a run duration is present.
521        pub fn with_any_run_duration(mut self) -> Self {
522            self.duration_milliseconds = MatchOption::Any;
523            self
524        }
525
526        /// Verify that no start time is present.
527        pub fn with_no_start_time(mut self) -> Self {
528            self.start_time = MatchOption::None;
529            self
530        }
531
532        /// Verify that no run duration is present.
533        pub fn with_no_run_duration(mut self) -> Self {
534            self.duration_milliseconds = MatchOption::None;
535            self
536        }
537    };
538}
539
540impl ExpectedTestRun {
541    /// Create a new `ExpectedTestRun` with the given `outcome`.
542    pub fn new(outcome: Outcome) -> Self {
543        Self {
544            artifacts: ArtifactMetadataToAssertionMap::new(),
545            outcome: outcome.into(),
546            start_time: MatchOption::AnyOrNone,
547            duration_milliseconds: MatchOption::AnyOrNone,
548            suites: vec![],
549        }
550    }
551
552    pub fn with_suite(mut self, suite: ExpectedSuite) -> Self {
553        self.suites.push(suite);
554        self
555    }
556
557    common_impl! {}
558}
559
560impl ExpectedSuite {
561    /// Create a new `ExpectedTestRun` with the given `name` and `outcome`.
562    pub fn new<S: AsRef<str>>(name: S, outcome: Outcome) -> Self {
563        Self {
564            artifacts: ArtifactMetadataToAssertionMap::new(),
565            name: name.as_ref().to_string(),
566            outcome: outcome.into(),
567            cases: HashMap::new(),
568            start_time: MatchOption::AnyOrNone,
569            duration_milliseconds: MatchOption::AnyOrNone,
570            tags: vec![],
571        }
572    }
573
574    /// Add a test case to the suite.
575    pub fn with_case(mut self, case: ExpectedTestCase) -> Self {
576        self.cases.insert(case.name.clone(), case);
577        self
578    }
579
580    /// Add a tag to the suite.
581    pub fn with_tag(mut self, tag: TestTag) -> Self {
582        self.tags.push(tag);
583        self
584    }
585
586    common_impl! {}
587}
588
589impl ExpectedTestCase {
590    /// Create a new `ExpectedTestCase` with the given `name` and `outcome`.
591    pub fn new<S: AsRef<str>>(name: S, outcome: Outcome) -> Self {
592        Self {
593            artifacts: ArtifactMetadataToAssertionMap::new(),
594            name: name.as_ref().to_string(),
595            outcome: outcome.into(),
596            start_time: MatchOption::AnyOrNone,
597            duration_milliseconds: MatchOption::AnyOrNone,
598        }
599    }
600
601    common_impl! {}
602}
603
604#[cfg(test)]
605mod test {
606    use super::*;
607    use crate::{ArtifactType, CommonResult, OutputDirectoryBuilder, SchemaVersion, RUN_NAME};
608    use std::borrow::Cow;
609    use std::io::Write;
610
611    fn test_with_directory<F: Fn(OutputDirectoryBuilder)>(_test_name: &str, test_fn: F) {
612        for version in SchemaVersion::all_variants() {
613            let dir = tempfile::TempDir::new().unwrap();
614            let directory_builder =
615                OutputDirectoryBuilder::new(dir.path(), version).expect("Create directory builder");
616            test_fn(directory_builder);
617        }
618    }
619
620    #[fixture::fixture(test_with_directory)]
621    #[test]
622    fn assert_run_result_check_outcome_only(output_dir: OutputDirectoryBuilder) {
623        let actual = TestRunResult {
624            common: Cow::Owned(CommonResult {
625                name: RUN_NAME.to_string(),
626                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
627                outcome: Outcome::Passed.into(),
628                start_time: Some(64),
629                duration_milliseconds: Some(128),
630            }),
631            suites: vec![],
632        };
633
634        output_dir.save_summary(&actual).expect("save summary");
635        assert_run_result(
636            output_dir.path(),
637            &ExpectedTestRun::new(Outcome::Passed).with_any_start_time().with_any_run_duration(),
638        );
639    }
640
641    #[fixture::fixture(test_with_directory)]
642    #[test]
643    fn assert_run_result_check_exact_timing(output_dir: OutputDirectoryBuilder) {
644        let actual = TestRunResult {
645            common: Cow::Owned(CommonResult {
646                name: RUN_NAME.to_string(),
647                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
648                outcome: Outcome::Passed.into(),
649                start_time: Some(64),
650                duration_milliseconds: Some(128),
651            }),
652            suites: vec![],
653        };
654
655        output_dir.save_summary(&actual).expect("save summary");
656        assert_run_result(
657            output_dir.path(),
658            &ExpectedTestRun::new(Outcome::Passed).with_start_time(64).with_run_duration(128),
659        );
660    }
661
662    #[fixture::fixture(test_with_directory)]
663    #[test]
664    fn assert_run_result_check_timing_unspecified(output_dir: OutputDirectoryBuilder) {
665        let actual = TestRunResult {
666            common: Cow::Owned(CommonResult {
667                name: RUN_NAME.to_string(),
668                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
669                outcome: Outcome::Passed.into(),
670                start_time: None,
671                duration_milliseconds: None,
672            }),
673            suites: vec![],
674        };
675
676        output_dir.save_summary(&actual).expect("save summary");
677        assert_run_result(
678            output_dir.path(),
679            &ExpectedTestRun::new(Outcome::Passed).with_no_start_time().with_no_run_duration(),
680        );
681    }
682
683    #[fixture::fixture(test_with_directory)]
684    #[test]
685    fn assert_run_result_single_artifact_unspecified_name(output_dir: OutputDirectoryBuilder) {
686        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
687        let mut artifact =
688            artifact_dir.new_artifact(ArtifactType::Syslog, "b.txt").expect("create artifact");
689        write!(artifact, "hello").expect("write to artifact");
690        drop(artifact);
691
692        let actual = TestRunResult {
693            common: Cow::Owned(CommonResult {
694                name: RUN_NAME.to_string(),
695                artifact_dir,
696                outcome: Outcome::Passed.into(),
697                start_time: None,
698                duration_milliseconds: None,
699            }),
700            suites: vec![],
701        };
702
703        output_dir.save_summary(&actual).expect("save summary");
704        assert_run_result(
705            output_dir.path(),
706            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
707                ArtifactType::Syslog,
708                Option::<&str>::None,
709                "hello",
710            ),
711        );
712    }
713
714    #[fixture::fixture(test_with_directory)]
715    #[test]
716    fn assert_run_result_single_artifact_specified_name(output_dir: OutputDirectoryBuilder) {
717        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
718        let mut artifact =
719            artifact_dir.new_artifact(ArtifactType::Syslog, "b.txt").expect("create artifact");
720        write!(artifact, "hello").expect("write to artifact");
721        drop(artifact);
722
723        let actual = TestRunResult {
724            common: Cow::Owned(CommonResult {
725                name: RUN_NAME.to_string(),
726                artifact_dir,
727                outcome: Outcome::Passed.into(),
728                start_time: None,
729                duration_milliseconds: None,
730            }),
731            suites: vec![],
732        };
733
734        output_dir.save_summary(&actual).expect("save summary");
735        assert_run_result(
736            output_dir.path(),
737            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
738                ArtifactType::Syslog,
739                "b.txt".into(),
740                "hello",
741            ),
742        );
743    }
744
745    #[fixture::fixture(test_with_directory)]
746    #[test]
747    #[should_panic(expected = "Outcome for TEST RUN")]
748    fn assert_run_outcome_mismatch(output_dir: OutputDirectoryBuilder) {
749        let actual = TestRunResult {
750            common: Cow::Owned(CommonResult {
751                name: RUN_NAME.to_string(),
752                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
753                outcome: Outcome::Failed.into(),
754                start_time: None,
755                duration_milliseconds: None,
756            }),
757            suites: vec![],
758        };
759
760        output_dir.save_summary(&actual).expect("save summary");
761        assert_run_result(output_dir.path(), &ExpectedTestRun::new(Outcome::Passed));
762    }
763
764    #[fixture::fixture(test_with_directory)]
765    #[test]
766    #[should_panic(expected = "Start time for TEST RUN")]
767    fn assert_run_start_time_mismatch(output_dir: OutputDirectoryBuilder) {
768        let actual = TestRunResult {
769            common: Cow::Owned(CommonResult {
770                name: RUN_NAME.to_string(),
771                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
772                outcome: Outcome::Failed.into(),
773                start_time: Some(64),
774                duration_milliseconds: None,
775            }),
776            suites: vec![],
777        };
778
779        output_dir.save_summary(&actual).expect("save summary");
780        assert_run_result(
781            output_dir.path(),
782            &ExpectedTestRun::new(Outcome::Passed).with_start_time(23),
783        );
784    }
785
786    #[fixture::fixture(test_with_directory)]
787    #[test]
788    #[should_panic(expected = "Run duration for TEST RUN")]
789    fn assert_run_duration_mismatch(output_dir: OutputDirectoryBuilder) {
790        let actual = TestRunResult {
791            common: Cow::Owned(CommonResult {
792                name: RUN_NAME.to_string(),
793                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
794                outcome: Outcome::Failed.into(),
795                start_time: None,
796                duration_milliseconds: None,
797            }),
798            suites: vec![],
799        };
800
801        output_dir.save_summary(&actual).expect("save summary");
802        assert_run_result(
803            output_dir.path(),
804            &ExpectedTestRun::new(Outcome::Passed).with_run_duration(23),
805        );
806    }
807
808    #[fixture::fixture(test_with_directory)]
809    #[test]
810    #[should_panic]
811    fn assert_run_artifact_mismatch(output_dir: OutputDirectoryBuilder) {
812        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
813        let mut artifact =
814            artifact_dir.new_artifact(ArtifactType::Syslog, "missing").expect("create artifact");
815        write!(artifact, "hello").expect("write to artifact");
816        drop(artifact);
817
818        let actual = TestRunResult {
819            common: Cow::Owned(CommonResult {
820                name: RUN_NAME.to_string(),
821                artifact_dir,
822                outcome: Outcome::Failed.into(),
823                start_time: None,
824                duration_milliseconds: None,
825            }),
826            suites: vec![],
827        };
828
829        output_dir.save_summary(&actual).expect("save summary");
830        assert_run_result(
831            output_dir.path(),
832            &ExpectedTestRun::new(Outcome::Failed).with_artifact(
833                ArtifactType::Stderr,
834                "stderr.txt".into(),
835                "",
836            ),
837        );
838    }
839
840    fn passing_run_with_single_suite<'a>(
841        output_dir: &OutputDirectoryBuilder,
842        suite: SuiteResult<'a>,
843    ) -> TestRunResult<'a> {
844        TestRunResult {
845            common: Cow::Owned(CommonResult {
846                name: RUN_NAME.to_string(),
847                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
848                outcome: Outcome::Passed.into(),
849                start_time: Some(64),
850                duration_milliseconds: Some(128),
851            }),
852            suites: vec![suite],
853        }
854    }
855
856    #[fixture::fixture(test_with_directory)]
857    #[test]
858    fn assert_run_result_with_suite(output_dir: OutputDirectoryBuilder) {
859        let actual = passing_run_with_single_suite(
860            &output_dir,
861            SuiteResult {
862                common: Cow::Owned(CommonResult {
863                    name: "suite".to_string(),
864                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
865                    outcome: Outcome::Passed.into(),
866                    start_time: Some(64),
867                    duration_milliseconds: Some(128),
868                }),
869                cases: vec![],
870                tags: Cow::Owned(vec![]),
871            },
872        );
873
874        output_dir.save_summary(&actual).expect("save summary");
875        assert_run_result(
876            output_dir.path(),
877            &ExpectedTestRun::new(Outcome::Passed)
878                .with_any_start_time()
879                .with_any_run_duration()
880                .with_suite(
881                    ExpectedSuite::new("suite", Outcome::Passed)
882                        .with_any_start_time()
883                        .with_any_run_duration(),
884                ),
885        );
886    }
887
888    #[fixture::fixture(test_with_directory)]
889    #[test]
890    fn assert_run_result_with_suite_exact_times(output_dir: OutputDirectoryBuilder) {
891        let actual = passing_run_with_single_suite(
892            &output_dir,
893            SuiteResult {
894                common: Cow::Owned(CommonResult {
895                    name: "suite".to_string(),
896                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
897                    outcome: Outcome::Passed.into(),
898                    start_time: Some(64),
899                    duration_milliseconds: Some(128),
900                }),
901                cases: vec![],
902                tags: Cow::Owned(vec![]),
903            },
904        );
905
906        output_dir.save_summary(&actual).expect("save summary");
907        assert_run_result(
908            output_dir.path(),
909            &ExpectedTestRun::new(Outcome::Passed)
910                .with_any_start_time()
911                .with_any_run_duration()
912                .with_suite(
913                    ExpectedSuite::new("suite", Outcome::Passed)
914                        .with_start_time(64)
915                        .with_run_duration(128),
916                ),
917        );
918    }
919
920    #[fixture::fixture(test_with_directory)]
921    #[test]
922    fn assert_run_result_with_suite_no_times(output_dir: OutputDirectoryBuilder) {
923        let actual = passing_run_with_single_suite(
924            &output_dir,
925            SuiteResult {
926                common: Cow::Owned(CommonResult {
927                    name: "suite".to_string(),
928                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
929                    outcome: Outcome::Passed.into(),
930                    start_time: None,
931                    duration_milliseconds: None,
932                }),
933                cases: vec![],
934                tags: Cow::Owned(vec![]),
935            },
936        );
937
938        output_dir.save_summary(&actual).expect("save summary");
939        assert_run_result(
940            output_dir.path(),
941            &ExpectedTestRun::new(Outcome::Passed)
942                .with_any_start_time()
943                .with_any_run_duration()
944                .with_suite(
945                    ExpectedSuite::new("suite", Outcome::Passed)
946                        .with_no_start_time()
947                        .with_no_run_duration(),
948                ),
949        );
950    }
951
952    #[fixture::fixture(test_with_directory)]
953    #[test]
954    fn assert_run_result_suite_with_artifact(output_dir: OutputDirectoryBuilder) {
955        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
956        let mut artifact =
957            artifact_dir.new_artifact(ArtifactType::Syslog, "b.txt").expect("create artifact");
958        write!(artifact, "hello").expect("write to artifact");
959        drop(artifact);
960
961        let actual = passing_run_with_single_suite(
962            &output_dir,
963            SuiteResult {
964                common: Cow::Owned(CommonResult {
965                    name: "suite".to_string(),
966                    artifact_dir,
967                    outcome: Outcome::Passed.into(),
968                    start_time: None,
969                    duration_milliseconds: None,
970                }),
971                cases: vec![],
972                tags: Cow::Owned(vec![]),
973            },
974        );
975
976        output_dir.save_summary(&actual).expect("save summary");
977        assert_run_result(
978            output_dir.path(),
979            &ExpectedTestRun::new(Outcome::Passed)
980                .with_any_start_time()
981                .with_any_run_duration()
982                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_artifact(
983                    ArtifactType::Syslog,
984                    "b.txt".into(),
985                    "hello",
986                )),
987        );
988    }
989
990    #[fixture::fixture(test_with_directory)]
991    #[test]
992    fn assert_run_result_suite_with_case(output_dir: OutputDirectoryBuilder) {
993        let actual = passing_run_with_single_suite(
994            &output_dir,
995            SuiteResult {
996                common: Cow::Owned(CommonResult {
997                    name: "suite".to_string(),
998                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
999                    outcome: Outcome::Passed.into(),
1000                    start_time: None,
1001                    duration_milliseconds: None,
1002                }),
1003                cases: vec![TestCaseResult {
1004                    common: Cow::Owned(CommonResult {
1005                        name: "case".to_string(),
1006                        artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1007                        outcome: Outcome::Passed.into(),
1008                        start_time: None,
1009                        duration_milliseconds: None,
1010                    }),
1011                }],
1012                tags: Cow::Owned(vec![]),
1013            },
1014        );
1015
1016        output_dir.save_summary(&actual).expect("save summary");
1017        assert_run_result(
1018            output_dir.path(),
1019            &ExpectedTestRun::new(Outcome::Passed)
1020                .with_any_start_time()
1021                .with_any_run_duration()
1022                .with_suite(
1023                    ExpectedSuite::new("suite", Outcome::Passed).with_case(
1024                        ExpectedTestCase::new("case", Outcome::Passed)
1025                            .with_no_run_duration()
1026                            .with_no_start_time(),
1027                    ),
1028                ),
1029        );
1030    }
1031
1032    #[fixture::fixture(test_with_directory)]
1033    #[test]
1034    fn assert_run_result_suite_with_tags(output_dir: OutputDirectoryBuilder) {
1035        let actual = passing_run_with_single_suite(
1036            &output_dir,
1037            SuiteResult {
1038                common: Cow::Owned(CommonResult {
1039                    name: "suite".to_string(),
1040                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1041                    outcome: Outcome::Passed.into(),
1042                    start_time: None,
1043                    duration_milliseconds: None,
1044                }),
1045                cases: vec![],
1046                tags: Cow::Owned(vec![
1047                    TestTag { key: "os".to_string(), value: "fuchsia".to_string() },
1048                    TestTag { key: "cpu".to_string(), value: "arm64".to_string() },
1049                ]),
1050            },
1051        );
1052
1053        output_dir.save_summary(&actual).expect("save summary");
1054        assert_run_result(
1055            output_dir.path(),
1056            &ExpectedTestRun::new(Outcome::Passed)
1057                .with_any_start_time()
1058                .with_any_run_duration()
1059                .with_suite(
1060                    ExpectedSuite::new("suite", Outcome::Passed)
1061                        .with_tag(TestTag { key: "cpu".to_string(), value: "arm64".to_string() })
1062                        .with_tag(TestTag { key: "os".to_string(), value: "fuchsia".to_string() }),
1063                ),
1064        );
1065    }
1066
1067    #[fixture::fixture(test_with_directory)]
1068    #[test]
1069    #[should_panic(expected = "Outcome for SUITE suite")]
1070    fn assert_suite_outcome_mismatch(output_dir: OutputDirectoryBuilder) {
1071        let actual = passing_run_with_single_suite(
1072            &output_dir,
1073            SuiteResult {
1074                common: Cow::Owned(CommonResult {
1075                    name: "suite".to_string(),
1076                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1077                    outcome: Outcome::Failed.into(),
1078                    start_time: None,
1079                    duration_milliseconds: None,
1080                }),
1081                cases: vec![],
1082                tags: Cow::Owned(vec![]),
1083            },
1084        );
1085
1086        output_dir.save_summary(&actual).expect("save summary");
1087        assert_run_result(
1088            output_dir.path(),
1089            &ExpectedTestRun::new(Outcome::Passed)
1090                .with_any_start_time()
1091                .with_any_run_duration()
1092                .with_suite(ExpectedSuite::new("suite", Outcome::Passed)),
1093        );
1094    }
1095
1096    #[fixture::fixture(test_with_directory)]
1097    #[test]
1098    #[should_panic(expected = "Start time for SUITE suite")]
1099    fn assert_suite_start_time_mismatch(output_dir: OutputDirectoryBuilder) {
1100        let actual = passing_run_with_single_suite(
1101            &output_dir,
1102            SuiteResult {
1103                common: Cow::Owned(CommonResult {
1104                    name: "suite".to_string(),
1105                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1106                    outcome: Outcome::Passed.into(),
1107                    start_time: None,
1108                    duration_milliseconds: Some(128),
1109                }),
1110                cases: vec![],
1111                tags: Cow::Owned(vec![]),
1112            },
1113        );
1114
1115        output_dir.save_summary(&actual).expect("save summary");
1116        assert_run_result(
1117            output_dir.path(),
1118            &ExpectedTestRun::new(Outcome::Passed)
1119                .with_any_start_time()
1120                .with_any_run_duration()
1121                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_any_start_time()),
1122        );
1123    }
1124
1125    #[fixture::fixture(test_with_directory)]
1126    #[test]
1127    #[should_panic(expected = "Duration for SUITE suite")]
1128    fn assert_suite_duration_mismatch(output_dir: OutputDirectoryBuilder) {
1129        let actual = passing_run_with_single_suite(
1130            &output_dir,
1131            SuiteResult {
1132                common: Cow::Owned(CommonResult {
1133                    name: "suite".to_string(),
1134                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1135                    outcome: Outcome::Passed.into(),
1136                    start_time: None,
1137                    duration_milliseconds: Some(128),
1138                }),
1139                cases: vec![],
1140                tags: Cow::Owned(vec![]),
1141            },
1142        );
1143
1144        output_dir.save_summary(&actual).expect("save summary");
1145        assert_run_result(
1146            output_dir.path(),
1147            &ExpectedTestRun::new(Outcome::Passed)
1148                .with_any_start_time()
1149                .with_any_run_duration()
1150                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_run_duration(32)),
1151        );
1152    }
1153
1154    #[fixture::fixture(test_with_directory)]
1155    #[test]
1156    #[should_panic]
1157    fn assert_suite_artifact_mismatch(output_dir: OutputDirectoryBuilder) {
1158        let actual = passing_run_with_single_suite(
1159            &output_dir,
1160            SuiteResult {
1161                common: Cow::Owned(CommonResult {
1162                    name: "suite".to_string(),
1163                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1164                    outcome: Outcome::Passed.into(),
1165                    start_time: None,
1166                    duration_milliseconds: Some(128),
1167                }),
1168                cases: vec![],
1169                tags: Cow::Owned(vec![]),
1170            },
1171        );
1172
1173        output_dir.save_summary(&actual).expect("save summary");
1174        assert_run_result(
1175            output_dir.path(),
1176            &ExpectedTestRun::new(Outcome::Passed)
1177                .with_any_start_time()
1178                .with_any_run_duration()
1179                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_artifact(
1180                    ArtifactType::Stderr,
1181                    Option::<&str>::None,
1182                    "missing contents",
1183                )),
1184        );
1185    }
1186
1187    #[fixture::fixture(test_with_directory)]
1188    #[test]
1189    #[should_panic(expected = "Found unexpected case")]
1190    fn assert_suite_case_mismatch(output_dir: OutputDirectoryBuilder) {
1191        let actual = passing_run_with_single_suite(
1192            &output_dir,
1193            SuiteResult {
1194                common: Cow::Owned(CommonResult {
1195                    name: "suite".to_string(),
1196                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1197                    outcome: Outcome::Failed.into(),
1198                    start_time: None,
1199                    duration_milliseconds: None,
1200                }),
1201                cases: vec![TestCaseResult {
1202                    common: Cow::Owned(CommonResult {
1203                        name: "case".to_string(),
1204                        artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1205                        outcome: Outcome::Passed.into(),
1206                        start_time: None,
1207                        duration_milliseconds: None,
1208                    }),
1209                }],
1210                tags: Cow::Owned(vec![]),
1211            },
1212        );
1213
1214        output_dir.save_summary(&actual).expect("save summary");
1215        assert_run_result(
1216            output_dir.path(),
1217            &ExpectedTestRun::new(Outcome::Passed).with_any_start_time().with_suite(
1218                ExpectedSuite::new("suite", Outcome::Failed)
1219                    .with_case(ExpectedTestCase::new("wrong name", Outcome::Passed)),
1220            ),
1221        );
1222    }
1223
1224    #[fixture::fixture(test_with_directory)]
1225    #[test]
1226    fn assert_artifacts_empty(output_dir: OutputDirectoryBuilder) {
1227        let actual = TestRunResult {
1228            common: Cow::Owned(CommonResult {
1229                name: RUN_NAME.to_string(),
1230                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1231                outcome: Outcome::Passed.into(),
1232                start_time: None,
1233                duration_milliseconds: None,
1234            }),
1235            suites: vec![],
1236        };
1237
1238        output_dir.save_summary(&actual).expect("save summary");
1239        assert_run_result(output_dir.path(), &ExpectedTestRun::new(Outcome::Passed));
1240    }
1241
1242    #[fixture::fixture(test_with_directory)]
1243    #[test]
1244    fn assert_artifacts_exact_content(output_dir: OutputDirectoryBuilder) {
1245        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1246        let mut artifact =
1247            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("new artifact");
1248        write!(artifact, "hello").expect("write to artifact");
1249        let actual = TestRunResult {
1250            common: Cow::Owned(CommonResult {
1251                name: RUN_NAME.to_string(),
1252                artifact_dir,
1253                outcome: Outcome::Passed.into(),
1254                start_time: None,
1255                duration_milliseconds: None,
1256            }),
1257            suites: vec![],
1258        };
1259
1260        output_dir.save_summary(&actual).expect("save summary");
1261        assert_run_result(
1262            output_dir.path(),
1263            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1264                ArtifactType::Stderr,
1265                Option::<&str>::None,
1266                "hello",
1267            ),
1268        );
1269    }
1270
1271    #[fixture::fixture(test_with_directory)]
1272    #[test]
1273    fn assert_artifacts_exact_content_exact_name(output_dir: OutputDirectoryBuilder) {
1274        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1275        let mut artifact =
1276            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("new artifact");
1277        write!(artifact, "hello").expect("write to artifact");
1278        let actual = TestRunResult {
1279            common: Cow::Owned(CommonResult {
1280                name: RUN_NAME.to_string(),
1281                artifact_dir,
1282                outcome: Outcome::Passed.into(),
1283                start_time: None,
1284                duration_milliseconds: None,
1285            }),
1286            suites: vec![],
1287        };
1288
1289        output_dir.save_summary(&actual).expect("save summary");
1290        assert_run_result(
1291            output_dir.path(),
1292            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1293                ArtifactType::Stderr,
1294                Some("b.txt"),
1295                "hello",
1296            ),
1297        );
1298    }
1299
1300    #[fixture::fixture(test_with_directory)]
1301    #[test]
1302    fn assert_artifacts_matching_content(output_dir: OutputDirectoryBuilder) {
1303        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1304        let mut artifact =
1305            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("new artifact");
1306        write!(artifact, "hello").expect("write to artifact");
1307        let actual = TestRunResult {
1308            common: Cow::Owned(CommonResult {
1309                name: RUN_NAME.to_string(),
1310                artifact_dir,
1311                outcome: Outcome::Passed.into(),
1312                start_time: None,
1313                duration_milliseconds: None,
1314            }),
1315            suites: vec![],
1316        };
1317
1318        output_dir.save_summary(&actual).expect("save summary");
1319        assert_run_result(
1320            output_dir.path(),
1321            &ExpectedTestRun::new(Outcome::Passed).with_matching_artifact(
1322                ArtifactType::Stderr,
1323                Some("b.txt"),
1324                |content| assert_eq!(content, "hello"),
1325            ),
1326        );
1327    }
1328
1329    #[fixture::fixture(test_with_directory)]
1330    #[test]
1331    fn assert_artifacts_moniker_specified(output_dir: OutputDirectoryBuilder) {
1332        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1333        let mut artifact = artifact_dir
1334            .new_artifact(
1335                ArtifactMetadata {
1336                    artifact_type: ArtifactType::Syslog.into(),
1337                    component_moniker: Some("moniker".into()),
1338                },
1339                "b.txt",
1340            )
1341            .expect("new artifact");
1342        write!(artifact, "hello").expect("write to artifact");
1343        let actual = TestRunResult {
1344            common: Cow::Owned(CommonResult {
1345                name: RUN_NAME.to_string(),
1346                artifact_dir,
1347                outcome: Outcome::Passed.into(),
1348                start_time: None,
1349                duration_milliseconds: None,
1350            }),
1351            suites: vec![],
1352        };
1353
1354        output_dir.save_summary(&actual).expect("save summary");
1355        assert_run_result(
1356            output_dir.path(),
1357            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1358                ArtifactMetadata {
1359                    artifact_type: ArtifactType::Syslog.into(),
1360                    component_moniker: Some("moniker".into()),
1361                },
1362                Some("b.txt"),
1363                "hello",
1364            ),
1365        );
1366    }
1367
1368    #[fixture::fixture(test_with_directory)]
1369    #[test]
1370    fn assert_artifacts_directory_artifact(output_dir: OutputDirectoryBuilder) {
1371        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1372        let dir_artifact =
1373            artifact_dir.new_directory_artifact(ArtifactType::Custom, "b").expect("new artifact");
1374        std::fs::write(dir_artifact.join("c.txt"), "hello c").unwrap();
1375        std::fs::write(dir_artifact.join("d.txt"), "hello d").unwrap();
1376        let actual = TestRunResult {
1377            common: Cow::Owned(CommonResult {
1378                name: RUN_NAME.to_string(),
1379                artifact_dir,
1380                outcome: Outcome::Passed.into(),
1381                start_time: None,
1382                duration_milliseconds: None,
1383            }),
1384            suites: vec![],
1385        };
1386
1387        output_dir.save_summary(&actual).expect("save summary");
1388        assert_run_result(
1389            output_dir.path(),
1390            &ExpectedTestRun::new(Outcome::Passed).with_directory_artifact(
1391                ArtifactType::Custom,
1392                Some("b"),
1393                ExpectedDirectory::new()
1394                    .with_file("c.txt", "hello c")
1395                    .with_matching_file("d.txt", |contents| assert_eq!(contents, "hello d")),
1396            ),
1397        );
1398    }
1399
1400    #[fixture::fixture(test_with_directory)]
1401    #[test]
1402    #[should_panic(expected = "Artifacts for TEST RUN")]
1403    fn assert_artifacts_missing(output_dir: OutputDirectoryBuilder) {
1404        let artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1405        let actual = TestRunResult {
1406            common: Cow::Owned(CommonResult {
1407                name: RUN_NAME.to_string(),
1408                artifact_dir,
1409                outcome: Outcome::Passed.into(),
1410                start_time: None,
1411                duration_milliseconds: None,
1412            }),
1413            suites: vec![],
1414        };
1415
1416        output_dir.save_summary(&actual).expect("save summary");
1417        assert_run_result(
1418            output_dir.path(),
1419            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1420                ArtifactType::Syslog,
1421                Some("missing"),
1422                "missing contents",
1423            ),
1424        );
1425    }
1426
1427    #[fixture::fixture(test_with_directory)]
1428    #[test]
1429    #[should_panic(expected = "Artifacts for TEST RUN")]
1430    fn assert_artifacts_extra_artifact(output_dir: OutputDirectoryBuilder) {
1431        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1432        let mut file_b =
1433            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("create artifact");
1434        write!(file_b, "hello").unwrap();
1435        let mut file_c =
1436            artifact_dir.new_artifact(ArtifactType::Stdout, "c.txt").expect("create artifact");
1437        write!(file_c, "hello").unwrap();
1438        drop(file_b);
1439        drop(file_c);
1440        let actual = TestRunResult {
1441            common: Cow::Owned(CommonResult {
1442                name: RUN_NAME.to_string(),
1443                artifact_dir,
1444                outcome: Outcome::Passed.into(),
1445                start_time: None,
1446                duration_milliseconds: None,
1447            }),
1448            suites: vec![],
1449        };
1450
1451        output_dir.save_summary(&actual).expect("save summary");
1452        assert_run_result(
1453            output_dir.path(),
1454            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1455                ArtifactType::Stderr,
1456                "c.txt".into(),
1457                "hello",
1458            ),
1459        );
1460    }
1461
1462    #[fixture::fixture(test_with_directory)]
1463    #[test]
1464    #[should_panic]
1465    fn assert_artifacts_content_not_equal(output_dir: OutputDirectoryBuilder) {
1466        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1467        let mut file_b =
1468            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("create artifact");
1469        write!(file_b, "wrong content").unwrap();
1470        drop(file_b);
1471        let actual = TestRunResult {
1472            common: Cow::Owned(CommonResult {
1473                name: RUN_NAME.to_string(),
1474                artifact_dir,
1475                outcome: Outcome::Passed.into(),
1476                start_time: None,
1477                duration_milliseconds: None,
1478            }),
1479            suites: vec![],
1480        };
1481
1482        output_dir.save_summary(&actual).expect("save summary");
1483        assert_run_result(
1484            output_dir.path(),
1485            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1486                ArtifactType::Syslog,
1487                Option::<&str>::None,
1488                "expected content",
1489            ),
1490        );
1491    }
1492
1493    #[fixture::fixture(test_with_directory)]
1494    #[test]
1495    #[should_panic]
1496    fn assert_artifacts_content_does_not_match(output_dir: OutputDirectoryBuilder) {
1497        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1498        let mut file_b =
1499            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("create artifact");
1500        write!(file_b, "wrong content").unwrap();
1501        drop(file_b);
1502        let actual = TestRunResult {
1503            common: Cow::Owned(CommonResult {
1504                name: RUN_NAME.to_string(),
1505                artifact_dir,
1506                outcome: Outcome::Passed.into(),
1507                start_time: None,
1508                duration_milliseconds: None,
1509            }),
1510            suites: vec![],
1511        };
1512
1513        output_dir.save_summary(&actual).expect("save summary");
1514        assert_run_result(
1515            output_dir.path(),
1516            &ExpectedTestRun::new(Outcome::Passed).with_matching_artifact(
1517                ArtifactType::Syslog,
1518                Option::<&str>::None,
1519                |content| assert_eq!(content, "expected content"),
1520            ),
1521        );
1522    }
1523
1524    #[fixture::fixture(test_with_directory)]
1525    #[test]
1526    #[should_panic]
1527    fn assert_artifacts_directory_mismatch(output_dir: OutputDirectoryBuilder) {
1528        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1529        let dir_artifact =
1530            artifact_dir.new_directory_artifact(ArtifactType::Custom, "b").expect("new artifact");
1531        std::fs::write(dir_artifact.join("c.txt"), "unexpected file").unwrap();
1532        let actual = TestRunResult {
1533            common: Cow::Owned(CommonResult {
1534                name: RUN_NAME.to_string(),
1535                artifact_dir,
1536                outcome: Outcome::Passed.into(),
1537                start_time: None,
1538                duration_milliseconds: None,
1539            }),
1540            suites: vec![],
1541        };
1542
1543        output_dir.save_summary(&actual).expect("save summary");
1544        assert_run_result(
1545            output_dir.path(),
1546            &ExpectedTestRun::new(Outcome::Passed).with_directory_artifact(
1547                ArtifactType::Custom,
1548                Option::<&str>::None,
1549                ExpectedDirectory::new(),
1550            ),
1551        );
1552    }
1553
1554    #[fixture::fixture(test_with_directory)]
1555    #[test]
1556    fn assert_artifacts_not_checked_if_unspecified(output_dir: OutputDirectoryBuilder) {
1557        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1558        let mut file_c =
1559            artifact_dir.new_artifact(ArtifactType::Stderr, "c.txt").expect("create artifact");
1560        write!(file_c, "unexpected file").unwrap();
1561        drop(file_c);
1562        let actual = TestRunResult {
1563            common: Cow::Owned(CommonResult {
1564                name: RUN_NAME.to_string(),
1565                artifact_dir,
1566                outcome: Outcome::Passed.into(),
1567                start_time: None,
1568                duration_milliseconds: None,
1569            }),
1570            suites: vec![],
1571        };
1572
1573        output_dir.save_summary(&actual).expect("save summary");
1574        assert_run_result(output_dir.path(), &ExpectedTestRun::new(Outcome::Passed));
1575    }
1576}
test_output_directory/testing.rs

test_output_directory/
testing.rs
