run_test_suite_lib/output/shell/
mod.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::output::noop::NoopDirectoryWriter;
6use crate::output::{
7    ArtifactType, DirectoryArtifactType, DynArtifact, DynDirectoryArtifact, EntityId, EntityInfo,
8    ReportedOutcome, Reporter, Timestamp,
9};
10use fuchsia_async as fasync;
11use fuchsia_sync::Mutex;
12use log::error;
13use std::collections::HashMap;
14use std::io::{Error, Write};
15use std::sync::atomic::AtomicU32;
16use std::sync::Arc;
17use std::time::Duration;
18
19mod writer;
20pub use writer::ShellWriterView;
21use writer::{ShellWriterHandle, ShellWriterHandleInner};
22
23/// Duration after which to emit an excessive duration log.
24const EXCESSIVE_DURATION: Duration = Duration::from_secs(60);
25/// Buffer stdout and stderr for this duration before dumping to console.
26const STDIO_BUFFERING_DURATION: Duration = Duration::from_secs(5);
27/// Dump stdout and stderr to console if it exceeds this size.
28const STDIO_BUFFER_SIZE: usize = 4096;
29
30/// A reporter that outputs results and artifacts to a single stream, usually stdout.
31/// This reporter is intended to provide "live" updates to a developer watching while
32/// tests are executed.
33pub struct ShellReporter<W: 'static + Write + Send + Sync> {
34    /// Arc around the writer and state, used to dispense more handles.
35    inner: Arc<Mutex<ShellWriterHandleInner<W>>>,
36    /// Map containing known information about each entity.
37    entity_state_map: Mutex<HashMap<EntityId, EntityState>>,
38    /// Number of completed suites, used to output
39    completed_suites: AtomicU32,
40    /// Number of suites expected in the run.
41    expected_suites: Mutex<Option<u32>>,
42    /// Size of the buffer used for stdio.
43    stdio_buffer_size: usize,
44    /// Length of time to buffer stdio before printing it to console.
45    stdio_buffer_duration: Duration,
46}
47
48/// All known state needed by a |ShellReporter| to display results.
49struct EntityState {
50    name: String,
51    excessive_duration_task: Option<fasync::Task<()>>,
52    children: Vec<EntityId>,
53    restricted_logs: Option<ShellWriterView<Vec<u8>>>,
54    run_state: EntityRunningState,
55}
56
57enum EntityRunningState {
58    NotRunning,
59    Started,
60    Finished(ReportedOutcome),
61}
62
63impl EntityState {
64    fn new<S: Into<String>>(name: S) -> Self {
65        Self {
66            name: name.into(),
67            excessive_duration_task: None,
68            children: vec![],
69            restricted_logs: None,
70            run_state: EntityRunningState::NotRunning,
71        }
72    }
73
74    fn name(&self) -> &str {
75        &self.name
76    }
77}
78
79impl ShellReporter<Vec<u8>> {
80    pub fn new_expose_writer_for_test() -> (Self, ShellWriterView<Vec<u8>>) {
81        let inner = Arc::new(Mutex::new(ShellWriterHandleInner::new(vec![])));
82        let mut entity_state_map = HashMap::new();
83        entity_state_map.insert(EntityId::TestRun, EntityState::new("TEST RUN"));
84        (
85            Self {
86                inner: inner.clone(),
87                entity_state_map: Mutex::new(entity_state_map),
88                completed_suites: AtomicU32::new(0),
89                expected_suites: Mutex::new(None),
90                // Disable buffering for most tests to simplify testing.
91                stdio_buffer_duration: Duration::ZERO,
92                stdio_buffer_size: 0,
93            },
94            ShellWriterView::new(inner),
95        )
96    }
97}
98
99impl<W: 'static + Write + Send + Sync> ShellReporter<W> {
100    pub fn new(inner: W) -> Self {
101        let inner = Arc::new(Mutex::new(ShellWriterHandleInner::new(inner)));
102        let mut entity_state_map = HashMap::new();
103        entity_state_map.insert(EntityId::TestRun, EntityState::new("TEST RUN"));
104        Self {
105            inner,
106            entity_state_map: Mutex::new(entity_state_map),
107            completed_suites: AtomicU32::new(0),
108            expected_suites: Mutex::new(None),
109            stdio_buffer_duration: STDIO_BUFFERING_DURATION,
110            stdio_buffer_size: STDIO_BUFFER_SIZE,
111        }
112    }
113
114    fn new_writer_handle(&self, prefix: Option<String>) -> ShellWriterHandle<W> {
115        ShellWriterHandle::new_handle(Arc::clone(&self.inner), prefix)
116    }
117}
118
119impl<W: 'static + Write + Send + Sync> Reporter for ShellReporter<W> {
120    fn new_entity(&self, entity: &EntityId, name: &str) -> Result<(), Error> {
121        let mut map = self.entity_state_map.lock();
122        map.insert(entity.clone(), EntityState::new(name));
123        if let EntityId::Case { .. } = entity {
124            map.get_mut(&EntityId::Suite).unwrap().children.push(entity.clone());
125        }
126        Ok(())
127    }
128
129    fn set_entity_info(&self, entity: &EntityId, info: &EntityInfo) {
130        match (entity, info.expected_children) {
131            (EntityId::TestRun, Some(children)) => {
132                self.expected_suites.lock().replace(children);
133            }
134            (_, _) => (),
135        }
136    }
137
138    fn entity_started(&self, entity: &EntityId, _: Timestamp) -> Result<(), Error> {
139        let mut writer = self.new_writer_handle(None);
140        let mut entity_map_lock = self.entity_state_map.lock();
141        let entity_entry = entity_map_lock.get_mut(entity).unwrap();
142        entity_entry.run_state = EntityRunningState::Started;
143        let name = entity_entry.name().to_string();
144        match entity {
145            EntityId::TestRun => (),
146            EntityId::Suite => writeln!(writer, "Running test '{}'", name)?,
147            EntityId::Case { .. } => {
148                writeln!(writer, "[RUNNING]\t{}", name)?;
149                entity_entry.excessive_duration_task = Some(fasync::Task::spawn(async move {
150                    fasync::Timer::new(EXCESSIVE_DURATION).await;
151                    writeln!(
152                        writer,
153                        "[duration - {}]:\tStill running after {:?} seconds",
154                        name,
155                        EXCESSIVE_DURATION.as_secs()
156                    )
157                    .unwrap_or_else(|e| error!("Failed to write: {:?}", e));
158                }));
159            }
160        }
161        Ok(())
162    }
163
164    fn entity_stopped(
165        &self,
166        entity: &EntityId,
167        outcome: &ReportedOutcome,
168        _: Timestamp,
169    ) -> Result<(), Error> {
170        let mut writer = self.new_writer_handle(None);
171        let mut entity_map_lock = self.entity_state_map.lock();
172        entity_map_lock.get_mut(entity).unwrap().run_state = EntityRunningState::Finished(*outcome);
173        let entity_entry = entity_map_lock.get_mut(entity).unwrap();
174        let name = entity_entry.name().to_string();
175
176        // cancel any tasks for reporting excessive duration
177        let _ = entity_entry.excessive_duration_task.take();
178        match entity {
179            EntityId::TestRun => (),
180            EntityId::Suite => (),
181            EntityId::Case { .. } => {
182                // We don't list error result as it indicates the test didn't finish.
183                if *outcome != ReportedOutcome::Error {
184                    writeln!(writer, "[{}]\t{}", outcome, name)?;
185                }
186            }
187        }
188        Ok(())
189    }
190
191    fn entity_finished(&self, entity: &EntityId) -> Result<(), Error> {
192        let mut writer = self.new_writer_handle(None);
193        let entity_map_lock = self.entity_state_map.lock();
194        let entity_entry = entity_map_lock.get(entity).unwrap();
195        let name = entity_entry.name().to_string();
196        let outcome = match &entity_entry.run_state {
197            EntityRunningState::Finished(outcome) => *outcome,
198            _ => ReportedOutcome::Inconclusive,
199        };
200        let children: Vec<_> = entity_entry.children.iter().cloned().collect();
201        match entity {
202            EntityId::TestRun => (),
203            EntityId::Suite => {
204                if matches!(entity_entry.run_state, EntityRunningState::NotRunning) {
205                    // no need to output a report if the test wasn't even started.
206                    return Ok(());
207                }
208
209                let cases: Vec<_> =
210                    children.iter().map(|child| entity_map_lock.get(child).unwrap()).collect();
211                let executed: Vec<_> = cases
212                    .iter()
213                    .filter(|case| match &case.run_state {
214                        EntityRunningState::Started => true,
215                        EntityRunningState::Finished(ReportedOutcome::Skipped) => false,
216                        EntityRunningState::Finished(_) => true,
217                        EntityRunningState::NotRunning => false,
218                    })
219                    .collect();
220                let mut failed: Vec<_> = cases
221                    .iter()
222                    .filter(|case| {
223                        matches!(
224                            &case.run_state,
225                            EntityRunningState::Finished(
226                                ReportedOutcome::Failed | ReportedOutcome::Timedout
227                            )
228                        )
229                    })
230                    .map(|case| case.name())
231                    .collect();
232                failed.sort();
233                let mut not_finished: Vec<_> = cases
234                    .iter()
235                    .filter(|case| {
236                        matches!(
237                            &case.run_state,
238                            EntityRunningState::Started
239                                | EntityRunningState::Finished(ReportedOutcome::Error)
240                        )
241                    })
242                    .map(|case| case.name())
243                    .collect();
244                not_finished.sort();
245                let num_passed = cases
246                    .iter()
247                    .filter(|case| {
248                        matches!(
249                            &case.run_state,
250                            EntityRunningState::Finished(ReportedOutcome::Passed)
251                        )
252                    })
253                    .count();
254                let num_skipped = cases
255                    .iter()
256                    .filter(|case| {
257                        matches!(
258                            &case.run_state,
259                            EntityRunningState::Finished(ReportedOutcome::Skipped)
260                        )
261                    })
262                    .count();
263
264                let suite_number =
265                    self.completed_suites.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
266                match &*self.expected_suites.lock() {
267                    Some(total_suites) if *total_suites > 1 => writeln!(
268                        writer,
269                        "\nTest suite count {}/{}",
270                        suite_number + 1,
271                        total_suites,
272                    )?,
273                    Some(_) | None => (),
274                }
275                writeln!(writer)?;
276                if !failed.is_empty() {
277                    writeln!(writer, "Failed tests: {}", failed.join(", "))?;
278                }
279                if !not_finished.is_empty() {
280                    writeln!(writer, "\nThe following test(s) never completed:")?;
281                    for t in not_finished {
282                        writeln!(writer, "{}", t)?;
283                    }
284                }
285                match num_skipped {
286                    0 => writeln!(
287                        writer,
288                        "{} out of {} tests passed...",
289                        num_passed,
290                        executed.len()
291                    )?,
292                    skipped => writeln!(
293                        writer,
294                        "{} out of {} attempted tests passed, {} tests skipped...",
295                        num_passed,
296                        executed.len(),
297                        skipped,
298                    )?,
299                }
300                if let Some(restricted_logs) = &entity_entry.restricted_logs {
301                    writeln!(writer, "\nTest {} produced unexpected high-severity logs:", &name)?;
302                    writeln!(writer, "----------------xxxxx----------------")?;
303                    writer.write_all(restricted_logs.lock().as_slice())?;
304                    writeln!(writer, "\n----------------xxxxx----------------")?;
305                    writeln!(writer, "Failing this test. See: https://fuchsia.dev/fuchsia-src/development/diagnostics/test_and_logs#restricting_log_severity\n")?;
306                }
307                match outcome {
308                    ReportedOutcome::Cancelled => {
309                        writeln!(writer, "{} was cancelled before completion.", &name)?
310                    }
311                    ReportedOutcome::DidNotFinish => {
312                        writeln!(writer, "{} did not complete successfully.", &name)?
313                    }
314                    other => writeln!(writer, "{} completed with result: {}", &name, other)?,
315                }
316            }
317            EntityId::Case { .. } => (),
318        }
319        Ok(())
320    }
321
322    fn new_artifact(
323        &self,
324        entity: &EntityId,
325        artifact_type: &ArtifactType,
326    ) -> Result<Box<DynArtifact>, Error> {
327        let mut lock = self.entity_state_map.lock();
328        let entity = lock.get_mut(entity).unwrap();
329        let name = entity.name();
330
331        Ok(match artifact_type {
332            ArtifactType::Stdout => Box::new(test_diagnostics::StdoutBuffer::new(
333                self.stdio_buffer_duration,
334                self.new_writer_handle(Some(format!("[stdout - {}]\n", name))),
335                self.stdio_buffer_size,
336            )),
337            ArtifactType::Stderr => Box::new(test_diagnostics::StdoutBuffer::new(
338                self.stdio_buffer_duration,
339                self.new_writer_handle(Some(format!("[stderr - {}]\n", name))),
340                self.stdio_buffer_size,
341            )),
342            ArtifactType::Syslog => Box::new(self.new_writer_handle(None)),
343            ArtifactType::RestrictedLog => {
344                // Restricted logs are saved for reporting when the entity completes.
345                let log_buffer = Arc::new(Mutex::new(ShellWriterHandleInner::new(vec![])));
346                entity.restricted_logs = Some(ShellWriterView::new(log_buffer.clone()));
347                Box::new(ShellWriterHandle::new_handle(log_buffer, None))
348            }
349        })
350    }
351
352    fn new_directory_artifact(
353        &self,
354        _entity: &EntityId,
355        _artifact_type: &DirectoryArtifactType,
356        _component_moniker: Option<String>,
357    ) -> Result<Box<DynDirectoryArtifact>, Error> {
358        // For the purpose of live reporting we don't display anything from a directory.
359        Ok(Box::new(NoopDirectoryWriter))
360    }
361}
362
363#[cfg(test)]
364mod test {
365    use super::*;
366    use crate::output::{CaseId, RunReporter};
367
368    #[fuchsia::test]
369    async fn report_case_events() {
370        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
371        let run_reporter = RunReporter::new(shell_reporter);
372        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
373        suite_reporter.started(Timestamp::Unknown).expect("case started");
374        let mut expected = "Running test 'test-suite'\n".to_string();
375        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
376
377        let case_1_reporter = suite_reporter.new_case("case-1", &CaseId(0)).expect("create case");
378        let case_2_reporter = suite_reporter.new_case("case-2", &CaseId(1)).expect("create case");
379
380        case_1_reporter.started(Timestamp::Unknown).expect("case started");
381        expected.push_str("[RUNNING]\tcase-1\n");
382        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
383
384        case_2_reporter.started(Timestamp::Unknown).expect("case started");
385        expected.push_str("[RUNNING]\tcase-2\n");
386        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
387
388        case_1_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
389        expected.push_str("[PASSED]\tcase-1\n");
390        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
391
392        case_2_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop case");
393        expected.push_str("[FAILED]\tcase-2\n");
394        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
395
396        case_1_reporter.finished().expect("finish case");
397        case_2_reporter.finished().expect("finish case");
398        suite_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop suite");
399        suite_reporter.finished().expect("finish suite");
400
401        expected.push_str("\n");
402        expected.push_str("Failed tests: case-2\n");
403        expected.push_str("1 out of 2 tests passed...\n");
404        expected.push_str("test-suite completed with result: FAILED\n");
405        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
406    }
407
408    #[fuchsia::test]
409    async fn report_multiple_suites() {
410        const NUM_SUITES: u32 = 5;
411        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
412        let run_reporter = RunReporter::new(shell_reporter);
413        run_reporter.set_expected_suites(NUM_SUITES);
414        run_reporter.started(Timestamp::Unknown).expect("run started");
415        let mut expected = "".to_string();
416
417        for suite_number in 0..4 {
418            let suite_name = format!("test-suite-{:?}", suite_number);
419            let suite_reporter = run_reporter.new_suite(&suite_name).expect("create suite");
420            suite_reporter.started(Timestamp::Unknown).expect("case started");
421            expected.push_str(&format!("Running test '{}'\n", &suite_name));
422            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
423
424            let case_reporter = suite_reporter.new_case("case-1", &CaseId(0)).expect("create case");
425            case_reporter.started(Timestamp::Unknown).expect("case started");
426            expected.push_str("[RUNNING]\tcase-1\n");
427            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
428            case_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
429            expected.push_str("[PASSED]\tcase-1\n");
430            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
431
432            case_reporter.finished().expect("finish case");
433            suite_reporter
434                .stopped(&ReportedOutcome::Passed, Timestamp::Unknown)
435                .expect("stop suite");
436            suite_reporter.finished().expect("finish suite");
437
438            expected.push_str("\n");
439            expected.push_str(&format!(
440                "Test suite count {:?}/{:?}\n\n",
441                suite_number + 1,
442                NUM_SUITES
443            ));
444            expected.push_str("1 out of 1 tests passed...\n");
445            expected.push_str(&format!("{} completed with result: PASSED\n", suite_name));
446            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
447        }
448    }
449
450    #[fuchsia::test]
451    async fn report_case_skipped() {
452        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
453        let run_reporter = RunReporter::new(shell_reporter);
454        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
455        suite_reporter.started(Timestamp::Unknown).expect("case started");
456        let mut expected = "Running test 'test-suite'\n".to_string();
457        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
458
459        let case_1_reporter = suite_reporter.new_case("case-1", &CaseId(0)).expect("create case");
460        let case_2_reporter = suite_reporter.new_case("case-2", &CaseId(1)).expect("create case");
461
462        case_1_reporter.started(Timestamp::Unknown).expect("case started");
463        expected.push_str("[RUNNING]\tcase-1\n");
464        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
465
466        case_1_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
467        expected.push_str("[PASSED]\tcase-1\n");
468        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
469
470        case_2_reporter.stopped(&ReportedOutcome::Skipped, Timestamp::Unknown).expect("stop case");
471        expected.push_str("[SKIPPED]\tcase-2\n");
472        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
473
474        case_1_reporter.finished().expect("finish case");
475        case_2_reporter.finished().expect("finish case");
476        suite_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop suite");
477        suite_reporter.finished().expect("finish suite");
478
479        expected.push_str("\n");
480        expected.push_str("1 out of 1 attempted tests passed, 1 tests skipped...\n");
481        expected.push_str("test-suite completed with result: PASSED\n");
482        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
483    }
484
485    #[fuchsia::test]
486    async fn syslog_artifacts() {
487        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
488        let run_reporter = RunReporter::new(shell_reporter);
489        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
490        suite_reporter.started(Timestamp::Unknown).expect("case started");
491        let mut syslog_writer =
492            suite_reporter.new_artifact(&ArtifactType::Syslog).expect("create syslog");
493
494        writeln!(syslog_writer, "[log] test syslog").expect("write to syslog");
495        let mut expected = "Running test 'test-suite'\n".to_string();
496        expected.push_str("[log] test syslog\n");
497        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected);
498
499        suite_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop suite");
500        writeln!(syslog_writer, "[log] more test syslog").expect("write to syslog");
501        expected.push_str("[log] more test syslog\n");
502        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected);
503
504        suite_reporter.finished().expect("finish suite");
505        expected.push_str("\n");
506        expected.push_str("0 out of 0 tests passed...\n");
507        expected.push_str("test-suite completed with result: PASSED\n");
508        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
509    }
510
511    #[fuchsia::test]
512    async fn report_retricted_logs() {
513        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
514        let run_reporter = RunReporter::new(shell_reporter);
515        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
516        suite_reporter.started(Timestamp::Unknown).expect("case started");
517
518        let case_reporter = suite_reporter.new_case("case-0", &CaseId(0)).expect("create case");
519        case_reporter.started(Timestamp::Unknown).expect("case started");
520        case_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
521        case_reporter.finished().expect("finish case");
522
523        let mut expected = "Running test 'test-suite'\n".to_string();
524        expected.push_str("[RUNNING]\tcase-0\n");
525        expected.push_str("[PASSED]\tcase-0\n");
526        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
527
528        suite_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop suite");
529        let mut restricted_log = suite_reporter
530            .new_artifact(&ArtifactType::RestrictedLog)
531            .expect("create restricted log");
532        write!(restricted_log, "suite restricted log").expect("write to restricted log");
533        drop(restricted_log);
534
535        suite_reporter.finished().expect("finish suite");
536        expected.push_str("\n");
537        expected.push_str("1 out of 1 tests passed...\n");
538        expected.push_str("\nTest test-suite produced unexpected high-severity logs:\n");
539        expected.push_str("----------------xxxxx----------------\n");
540        expected.push_str("suite restricted log\n\n");
541        expected.push_str("----------------xxxxx----------------\n");
542        expected.push_str("Failing this test. See: https://fuchsia.dev/fuchsia-src/development/diagnostics/test_and_logs#restricting_log_severity\n");
543        expected.push_str("\ntest-suite completed with result: FAILED\n");
544        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
545    }
546
547    #[fuchsia::test]
548    async fn stdout_artifacts() {
549        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
550        let run_reporter = RunReporter::new(shell_reporter);
551        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
552        suite_reporter.started(Timestamp::Unknown).expect("case started");
553
554        let case_0_reporter = suite_reporter.new_case("case-0", &CaseId(0)).expect("create case");
555        let case_1_reporter = suite_reporter.new_case("case-1", &CaseId(1)).expect("create case");
556        case_0_reporter.started(Timestamp::Unknown).expect("start case");
557        case_1_reporter.started(Timestamp::Unknown).expect("start case");
558        let mut expected = "Running test 'test-suite'\n".to_string();
559        expected.push_str("[RUNNING]\tcase-0\n[RUNNING]\tcase-1\n");
560        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
561
562        let mut case_0_stdout =
563            case_0_reporter.new_artifact(&ArtifactType::Stdout).expect("create artifact");
564        let mut case_1_stdout =
565            case_1_reporter.new_artifact(&ArtifactType::Stdout).expect("create artifact");
566
567        writeln!(case_0_stdout, "stdout from case 0").expect("write to stdout");
568        writeln!(case_1_stdout, "stdout from case 1").expect("write to stdout");
569
570        expected.push_str("[stdout - case-0]\nstdout from case 0\n");
571        expected.push_str("[stdout - case-1]\nstdout from case 1\n");
572        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
573
574        case_0_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
575        case_1_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
576        expected.push_str("[PASSED]\tcase-0\n");
577        expected.push_str("[PASSED]\tcase-1\n");
578        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
579
580        case_0_reporter.finished().expect("finish case");
581        case_1_reporter.finished().expect("finish case");
582        suite_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop suite");
583
584        suite_reporter.finished().expect("finish suite");
585        expected.push_str("\n");
586        expected.push_str("2 out of 2 tests passed...\n");
587        expected.push_str("test-suite completed with result: PASSED\n");
588        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
589    }
590
591    #[fuchsia::test]
592    async fn report_unfinished() {
593        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
594        let run_reporter = RunReporter::new(shell_reporter);
595        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
596        suite_reporter.started(Timestamp::Unknown).expect("suite started");
597
598        let case_reporter = suite_reporter.new_case("case-0", &CaseId(0)).expect("create case");
599        case_reporter.started(Timestamp::Unknown).expect("case started");
600        case_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
601        case_reporter.finished().expect("finish case");
602        let mut expected = "Running test 'test-suite'\n".to_string();
603        expected.push_str("[RUNNING]\tcase-0\n[PASSED]\tcase-0\n");
604        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
605
606        // add a case that finishes with ERROR
607        let unfinished_case_1 = suite_reporter.new_case("case-1", &CaseId(1)).expect("create case");
608        unfinished_case_1.started(Timestamp::Unknown).expect("case started");
609        unfinished_case_1.stopped(&ReportedOutcome::Error, Timestamp::Unknown).expect("stop case");
610        unfinished_case_1.finished().expect("finish case");
611        expected.push_str("[RUNNING]\tcase-1\n");
612        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
613
614        // add a case that doesn't finish at all
615        let unfinished_case_2 = suite_reporter.new_case("case-2", &CaseId(2)).expect("create case");
616        unfinished_case_2.started(Timestamp::Unknown).expect("case started");
617        unfinished_case_2.finished().expect("finish case");
618        expected.push_str("[RUNNING]\tcase-2\n");
619        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
620
621        suite_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop suite");
622        suite_reporter.finished().expect("finish suite");
623        expected.push_str("\n");
624        expected.push_str("\nThe following test(s) never completed:\n");
625        expected.push_str("case-1\n");
626        expected.push_str("case-2\n");
627        expected.push_str("1 out of 3 tests passed...\n");
628        expected.push_str("test-suite completed with result: FAILED\n");
629        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
630    }
631
632    #[fuchsia::test]
633    async fn report_cancelled_suite() {
634        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
635        let run_reporter = RunReporter::new(shell_reporter);
636        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
637        suite_reporter.started(Timestamp::Unknown).expect("suite started");
638
639        let case_reporter = suite_reporter.new_case("case", &CaseId(0)).expect("create new case");
640        case_reporter.started(Timestamp::Unknown).expect("case started");
641        case_reporter.finished().expect("case finished");
642        suite_reporter
643            .stopped(&ReportedOutcome::Cancelled, Timestamp::Unknown)
644            .expect("stop suite");
645        suite_reporter.finished().expect("case finished");
646
647        let mut expected = "Running test 'test-suite'\n".to_string();
648        expected.push_str("[RUNNING]\tcase\n");
649        expected.push_str("\n");
650        expected.push_str("\nThe following test(s) never completed:\n");
651        expected.push_str("case\n");
652        expected.push_str("0 out of 1 tests passed...\n");
653        expected.push_str("test-suite was cancelled before completion.\n");
654        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
655    }
656
657    #[fuchsia::test]
658    async fn report_suite_did_not_finish() {
659        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
660        let run_reporter = RunReporter::new(shell_reporter);
661        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
662        suite_reporter.started(Timestamp::Unknown).expect("suite started");
663
664        let case_reporter = suite_reporter.new_case("case", &CaseId(0)).expect("create new case");
665        case_reporter.started(Timestamp::Unknown).expect("case started");
666        case_reporter.finished().expect("case finished");
667        suite_reporter
668            .stopped(&ReportedOutcome::DidNotFinish, Timestamp::Unknown)
669            .expect("stop suite");
670        suite_reporter.finished().expect("case finished");
671
672        let mut expected = "Running test 'test-suite'\n".to_string();
673        expected.push_str("[RUNNING]\tcase\n");
674        expected.push_str("\n");
675        expected.push_str("\nThe following test(s) never completed:\n");
676        expected.push_str("case\n");
677        expected.push_str("0 out of 1 tests passed...\n");
678        expected.push_str("test-suite did not complete successfully.\n");
679        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
680    }
681}
run_test_suite_lib/output/shell/mod.rs

run_test_suite_lib/output/shell/
mod.rs
