test_manager_test_lib/
test_lib.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! This crate provides helper functions for testing architecture tests.
6
7use anyhow::{Context as _, Error, bail};
8use fidl_fuchsia_io as fio;
9use fidl_fuchsia_test_manager::{
10    self as ftest_manager, EventDetailsUnknown, SuiteControllerProxy, SuiteEvent as FidlSuiteEvent,
11    SuiteEventPayload as FidlSuiteEventPayload, SuiteEventPayloadUnknown,
12};
13use fuchsia_async as fasync;
14use futures::channel::mpsc;
15use futures::prelude::*;
16use linked_hash_map::LinkedHashMap;
17use log::*;
18use moniker::ExtendedMoniker;
19use std::collections::HashMap;
20use std::sync::Arc;
21use test_diagnostics::zstd_compress::Decoder;
22use test_diagnostics::{LogStream, collect_and_send_string_output, collect_string_from_socket};
23
24pub fn default_run_option() -> ftest_manager::RunOptions {
25    ftest_manager::RunOptions {
26        parallel: None,
27        arguments: None,
28        run_disabled_tests: Some(false),
29        timeout: None,
30        case_filters_to_run: None,
31        log_iterator: None,
32        ..Default::default()
33    }
34}
35
36pub fn default_run_suite_options() -> ftest_manager::RunSuiteOptions {
37    ftest_manager::RunSuiteOptions { run_disabled_tests: Some(false), ..Default::default() }
38}
39
40#[derive(Debug, Eq, PartialEq)]
41pub struct AttributedLog {
42    pub log: String,
43    pub moniker: ExtendedMoniker,
44}
45
46pub async fn collect_suite_events(
47    suite_instance: SuiteRunInstance,
48) -> Result<(Vec<RunEvent>, Vec<AttributedLog>), Error> {
49    let (sender, mut recv) = mpsc::channel(1);
50    let execution_task =
51        fasync::Task::spawn(async move { suite_instance.collect_events(sender).await });
52    let mut events = vec![];
53    let mut log_tasks = vec![];
54    while let Some(event) = recv.next().await {
55        match event.payload {
56            SuiteEventPayload::RunEvent(RunEvent::CaseStdout { name, mut stdout_message }) => {
57                if stdout_message.ends_with("\n") {
58                    stdout_message.truncate(stdout_message.len() - 1)
59                }
60                let logs = stdout_message.split("\n");
61                for log in logs {
62                    // gtest produces this line when tests are randomized. As of
63                    // this writing, our gtest_main binary *always* randomizes.
64                    if log.contains("Note: Randomizing tests' orders with a seed of") {
65                        continue;
66                    }
67                    events.push(RunEvent::case_stdout(name.clone(), log.to_string()));
68                }
69            }
70            SuiteEventPayload::RunEvent(RunEvent::CaseStderr { name, mut stderr_message }) => {
71                if stderr_message.ends_with("\n") {
72                    stderr_message.truncate(stderr_message.len() - 1)
73                }
74                let logs = stderr_message.split("\n");
75                for log in logs {
76                    events.push(RunEvent::case_stderr(name.clone(), log.to_string()));
77                }
78            }
79            SuiteEventPayload::RunEvent(e) => events.push(e),
80            SuiteEventPayload::SuiteLog { log_stream } => {
81                let t = fasync::Task::spawn(log_stream.collect::<Vec<_>>());
82                log_tasks.push(t);
83            }
84            SuiteEventPayload::TestCaseLog { .. } => {
85                panic!("not supported yet!")
86            }
87            SuiteEventPayload::DebugData { .. } => {
88                panic!("not supported yet!")
89            }
90        }
91    }
92    execution_task.await.context("test execution failed")?;
93
94    let mut collected_logs = vec![];
95    for t in log_tasks {
96        let logs = t.await;
97        for log_result in logs {
98            let log = log_result?;
99            collected_logs
100                .push(AttributedLog { log: log.msg().unwrap().to_string(), moniker: log.moniker });
101        }
102    }
103
104    Ok((events, collected_logs))
105}
106
107pub async fn collect_suite_events_with_watch(
108    suite_instance: SuiteRunInstance,
109    filter_debug_data: bool,
110    compressed_debug_data: bool,
111) -> Result<(Vec<RunEvent>, Vec<AttributedLog>), Error> {
112    let (sender, mut recv) = mpsc::channel(1);
113    let execution_task = fasync::Task::spawn(async move {
114        suite_instance
115            .collect_events_with_watch(sender, filter_debug_data, compressed_debug_data)
116            .await
117    });
118    let mut events = vec![];
119    let mut log_tasks = vec![];
120    while let Some(event) = recv.next().await {
121        match event.payload {
122            SuiteEventPayload::RunEvent(RunEvent::CaseStdout { name, mut stdout_message }) => {
123                if stdout_message.ends_with("\n") {
124                    stdout_message.truncate(stdout_message.len() - 1)
125                }
126                let logs = stdout_message.split("\n");
127                for log in logs {
128                    // gtest produces this line when tests are randomized. As of
129                    // this writing, our gtest_main binary *always* randomizes.
130                    if log.contains("Note: Randomizing tests' orders with a seed of") {
131                        continue;
132                    }
133                    events.push(RunEvent::case_stdout(name.clone(), log.to_string()));
134                }
135            }
136            SuiteEventPayload::RunEvent(RunEvent::CaseStderr { name, mut stderr_message }) => {
137                if stderr_message.ends_with("\n") {
138                    stderr_message.truncate(stderr_message.len() - 1)
139                }
140                let logs = stderr_message.split("\n");
141                for log in logs {
142                    events.push(RunEvent::case_stderr(name.clone(), log.to_string()));
143                }
144            }
145            SuiteEventPayload::RunEvent(e) => events.push(e),
146            SuiteEventPayload::SuiteLog { log_stream } => {
147                let t = fasync::Task::spawn(log_stream.collect::<Vec<_>>());
148                log_tasks.push(t);
149            }
150            SuiteEventPayload::TestCaseLog { .. } => {
151                panic!("not supported yet!")
152            }
153            SuiteEventPayload::DebugData { filename, socket } => {
154                events.push(RunEvent::DebugData { filename, socket })
155            }
156        }
157    }
158    execution_task.await.context("test execution failed")?;
159
160    let mut collected_logs = vec![];
161    for t in log_tasks {
162        let logs = t.await;
163        for log_result in logs {
164            let log = log_result?;
165            collected_logs
166                .push(AttributedLog { log: log.msg().unwrap().to_string(), moniker: log.moniker });
167        }
168    }
169
170    Ok((events, collected_logs))
171}
172
173/// Collect bytes from the socket, decompress if required and return the string
174pub async fn collect_string_from_socket_helper(
175    socket: fidl::Socket,
176    compressed_debug_data: bool,
177) -> Result<String, anyhow::Error> {
178    if !compressed_debug_data {
179        return collect_string_from_socket(socket).await;
180    }
181    let mut async_socket = fidl::AsyncSocket::from_socket(socket);
182    let mut buf = vec![0u8; 1024 * 32];
183
184    let (mut decoder, mut receiver) = Decoder::new();
185    let task: fasync::Task<Result<(), anyhow::Error>> = fasync::Task::spawn(async move {
186        loop {
187            let l = async_socket.read(&mut buf).await?;
188            match l {
189                0 => {
190                    decoder.finish().await?;
191                    break;
192                }
193                _ => {
194                    decoder.decompress(&buf[..l]).await?;
195                }
196            }
197        }
198        Ok(())
199    });
200
201    let mut decompressed_data = Vec::new();
202    while let Some(chunk) = receiver.next().await {
203        decompressed_data.extend_from_slice(&chunk);
204    }
205    task.await?;
206    Ok(String::from_utf8_lossy(decompressed_data.as_slice()).into())
207}
208/// Runs a test suite.
209pub struct SuiteRunner {
210    proxy: ftest_manager::SuiteRunnerProxy,
211}
212
213impl SuiteRunner {
214    /// Create new instance
215    pub fn new(proxy: ftest_manager::SuiteRunnerProxy) -> Self {
216        Self { proxy }
217    }
218
219    pub fn take_proxy(self) -> ftest_manager::SuiteRunnerProxy {
220        self.proxy
221    }
222
223    /// Starts the suite run, returning the suite run controller wrapped in a SuiteRunInstance.
224    pub fn start_suite_run(
225        &self,
226        test_url: &str,
227        options: ftest_manager::RunSuiteOptions,
228    ) -> Result<SuiteRunInstance, Error> {
229        let (controller_proxy, controller) = fidl::endpoints::create_proxy();
230        self.proxy.run(test_url, options, controller).context("Error starting tests")?;
231
232        return Ok(SuiteRunInstance { controller_proxy: controller_proxy.into() });
233    }
234}
235
236/// Builds and runs test suite(s).
237pub struct TestBuilder {
238    proxy: ftest_manager::RunBuilderProxy,
239    filter_debug_data: bool,
240}
241
242impl TestBuilder {
243    /// Create new instance
244    pub fn new(proxy: ftest_manager::RunBuilderProxy) -> Self {
245        Self { proxy, filter_debug_data: false }
246    }
247
248    /// Filter out debug data. On coverage builders, tests executed under
249    /// test_manager produce coverage profile. This option is useful for
250    /// ignoring these and ensuring the caller observes the same events on
251    /// all builders.
252    pub fn filter_debug_data(self) -> Self {
253        let Self { proxy, .. } = self;
254        Self { proxy, filter_debug_data: true }
255    }
256
257    pub fn take_proxy(self) -> ftest_manager::RunBuilderProxy {
258        self.proxy
259    }
260
261    pub fn set_scheduling_options(&self, accumulate_debug_data: bool) -> Result<(), Error> {
262        self.proxy
263            .with_scheduling_options(&ftest_manager::SchedulingOptions {
264                accumulate_debug_data: Some(accumulate_debug_data),
265                ..Default::default()
266            })
267            .map_err(Error::from)
268    }
269
270    /// Add suite to run.
271    pub async fn add_suite(
272        &self,
273        test_url: &str,
274        run_options: ftest_manager::RunOptions,
275    ) -> Result<SuiteRunInstance, Error> {
276        let (controller_proxy, controller) = fidl::endpoints::create_proxy();
277        self.proxy.add_suite(test_url, &run_options, controller)?;
278        Ok(SuiteRunInstance { controller_proxy: controller_proxy.into() })
279    }
280
281    /// Add suite to run in a realm.
282    pub async fn add_suite_in_realm(
283        &self,
284        realm: fidl::endpoints::ClientEnd<fidl_fuchsia_component::RealmMarker>,
285        offers: &[fidl_fuchsia_component_decl::Offer],
286        test_collection: &str,
287        test_url: &str,
288        run_options: ftest_manager::RunOptions,
289    ) -> Result<SuiteRunInstance, Error> {
290        let (controller_proxy, controller) = fidl::endpoints::create_proxy();
291        self.proxy.add_suite_in_realm(
292            realm,
293            offers,
294            test_collection,
295            test_url,
296            &run_options,
297            controller,
298        )?;
299        Ok(SuiteRunInstance { controller_proxy: controller_proxy.into() })
300    }
301
302    /// Runs all tests to completion and collects events alongside uncompressed debug_data.
303    /// We will remove this function and merge it with run_with_option function once we remove open to get uncompressed
304    /// debug_data.
305    pub async fn run(self) -> Result<Vec<TestRunEvent>, Error> {
306        self.run_with_option(false).await
307    }
308
309    /// Runs all tests to completion and collects events alongside compressed debug_data.
310    /// We will remove this function and merge it with run once we remove open to get uncompressed
311    /// debug_data.
312    pub async fn run_with_option(
313        self,
314        get_compressed_debug_data: bool,
315    ) -> Result<Vec<TestRunEvent>, Error> {
316        let (controller_proxy, controller) = fidl::endpoints::create_proxy();
317        self.proxy.build(controller).context("Error starting tests")?;
318        // wait for test to end
319        let mut events = vec![];
320        loop {
321            let fidl_events = controller_proxy.get_events().await.context("Get run events")?;
322            if fidl_events.is_empty() {
323                break;
324            }
325            for fidl_event in fidl_events {
326                match fidl_event.payload.expect("Details cannot be empty") {
327                    ftest_manager::RunEventPayload::Artifact(
328                        ftest_manager::Artifact::DebugData(iterator),
329                    ) => {
330                        if !self.filter_debug_data {
331                            let proxy = iterator.into_proxy();
332                            loop {
333                                let data = match get_compressed_debug_data {
334                                    true => proxy.get_next_compressed().await?,
335                                    false => proxy.get_next().await?,
336                                };
337                                if data.is_empty() {
338                                    break;
339                                }
340                                for data_file in data {
341                                    let socket = data_file.socket.expect("File cannot be empty");
342                                    events.push(TestRunEvent::debug_data(
343                                        fidl_event.timestamp,
344                                        data_file.name.expect("Name cannot be empty"),
345                                        socket,
346                                    ));
347                                }
348                            }
349                        }
350                    }
351                    other => bail!("Expected only debug data run events but got {:?}", other),
352                }
353            }
354        }
355        Ok(events)
356    }
357}
358
359#[derive(Debug)]
360pub struct TestRunEvent {
361    pub timestamp: Option<i64>,
362    pub payload: TestRunEventPayload,
363}
364
365impl TestRunEvent {
366    pub fn debug_data<S: Into<String>>(
367        timestamp: Option<i64>,
368        filename: S,
369        socket: fidl::Socket,
370    ) -> Self {
371        Self {
372            timestamp,
373            payload: TestRunEventPayload::DebugData { filename: filename.into(), socket },
374        }
375    }
376}
377
378#[derive(Debug)]
379pub enum TestRunEventPayload {
380    DebugData { filename: String, socket: fidl::Socket },
381}
382
383/// Events produced by test suite.
384pub struct SuiteEvent {
385    pub timestamp: Option<i64>,
386    pub payload: SuiteEventPayload,
387}
388
389impl SuiteEvent {
390    // Note: This is only used with SuiteRunner, not RunBuilder.
391    pub fn debug_data<S: Into<String>>(
392        timestamp: Option<i64>,
393        filename: S,
394        socket: fidl::Socket,
395    ) -> Self {
396        Self {
397            timestamp,
398            payload: SuiteEventPayload::DebugData { filename: filename.into(), socket },
399        }
400    }
401
402    pub fn case_found(timestamp: Option<i64>, name: String) -> Self {
403        SuiteEvent { timestamp, payload: SuiteEventPayload::RunEvent(RunEvent::case_found(name)) }
404    }
405
406    pub fn case_started(timestamp: Option<i64>, name: String) -> Self {
407        SuiteEvent { timestamp, payload: SuiteEventPayload::RunEvent(RunEvent::case_started(name)) }
408    }
409
410    pub fn case_stdout<N, L>(timestamp: Option<i64>, name: N, stdout_message: L) -> Self
411    where
412        N: Into<String>,
413        L: Into<String>,
414    {
415        SuiteEvent {
416            timestamp,
417            payload: SuiteEventPayload::RunEvent(RunEvent::case_stdout(
418                name.into(),
419                stdout_message.into(),
420            )),
421        }
422    }
423
424    pub fn case_stderr<N, L>(timestamp: Option<i64>, name: N, stderr_message: L) -> Self
425    where
426        N: Into<String>,
427        L: Into<String>,
428    {
429        SuiteEvent {
430            timestamp,
431            payload: SuiteEventPayload::RunEvent(RunEvent::case_stderr(
432                name.into(),
433                stderr_message.into(),
434            )),
435        }
436    }
437
438    pub fn case_stopped(
439        timestamp: Option<i64>,
440        name: String,
441        status: ftest_manager::TestCaseResult,
442    ) -> Self {
443        SuiteEvent {
444            timestamp,
445            payload: SuiteEventPayload::RunEvent(RunEvent::case_stopped(name, status)),
446        }
447    }
448
449    pub fn case_finished(timestamp: Option<i64>, name: String) -> Self {
450        SuiteEvent {
451            timestamp,
452            payload: SuiteEventPayload::RunEvent(RunEvent::case_finished(name)),
453        }
454    }
455
456    pub fn suite_stopped(timestamp: Option<i64>, status: ftest_manager::SuiteResult) -> Self {
457        SuiteEvent {
458            timestamp,
459            payload: SuiteEventPayload::RunEvent(RunEvent::suite_stopped(status)),
460        }
461    }
462
463    pub fn suite_custom(
464        timestamp: Option<i64>,
465        component: String,
466        filename: String,
467        contents: String,
468    ) -> Self {
469        SuiteEvent {
470            timestamp,
471            payload: SuiteEventPayload::RunEvent(RunEvent::suite_custom(
472                component, filename, contents,
473            )),
474        }
475    }
476
477    pub fn suite_log(timestamp: Option<i64>, log_stream: LogStream) -> Self {
478        SuiteEvent { timestamp, payload: SuiteEventPayload::SuiteLog { log_stream } }
479    }
480
481    pub fn test_case_log(timestamp: Option<i64>, name: String, log_stream: LogStream) -> Self {
482        SuiteEvent { timestamp, payload: SuiteEventPayload::TestCaseLog { name, log_stream } }
483    }
484}
485
486pub enum SuiteEventPayload {
487    /// Logger for test suite
488    SuiteLog {
489        log_stream: LogStream,
490    },
491
492    /// Logger for a test case in suite.
493    TestCaseLog {
494        name: String,
495        log_stream: LogStream,
496    },
497
498    /// Test events.
499    RunEvent(RunEvent),
500
501    // Debug data. Note: This is only used with SuiteRunner, not RunBuilder.
502    DebugData {
503        filename: String,
504        socket: fidl::Socket,
505    },
506}
507
508#[derive(PartialEq, Debug, Eq, Hash, Ord, PartialOrd)]
509pub enum RunEvent {
510    CaseFound { name: String },
511    CaseStarted { name: String },
512    CaseStdout { name: String, stdout_message: String },
513    CaseStderr { name: String, stderr_message: String },
514    CaseStopped { name: String, status: ftest_manager::TestCaseResult },
515    CaseFinished { name: String },
516    SuiteStarted,
517    SuiteCustom { component: String, filename: String, contents: String },
518    SuiteStopped { status: ftest_manager::SuiteResult },
519    DebugData { filename: String, socket: fidl::Socket },
520}
521
522impl RunEvent {
523    pub fn case_found<S>(name: S) -> Self
524    where
525        S: Into<String>,
526    {
527        Self::CaseFound { name: name.into() }
528    }
529
530    pub fn case_started<S>(name: S) -> Self
531    where
532        S: Into<String>,
533    {
534        Self::CaseStarted { name: name.into() }
535    }
536
537    pub fn case_stdout<S, L>(name: S, stdout_message: L) -> Self
538    where
539        S: Into<String>,
540        L: Into<String>,
541    {
542        Self::CaseStdout { name: name.into(), stdout_message: stdout_message.into() }
543    }
544
545    pub fn case_stderr<S, L>(name: S, stderr_message: L) -> Self
546    where
547        S: Into<String>,
548        L: Into<String>,
549    {
550        Self::CaseStderr { name: name.into(), stderr_message: stderr_message.into() }
551    }
552
553    pub fn case_stopped<S>(name: S, status: ftest_manager::TestCaseResult) -> Self
554    where
555        S: Into<String>,
556    {
557        Self::CaseStopped { name: name.into(), status }
558    }
559
560    pub fn case_finished<S>(name: S) -> Self
561    where
562        S: Into<String>,
563    {
564        Self::CaseFinished { name: name.into() }
565    }
566
567    pub fn suite_started() -> Self {
568        Self::SuiteStarted
569    }
570
571    pub fn suite_custom<T, U, V>(component: T, filename: U, contents: V) -> Self
572    where
573        T: Into<String>,
574        U: Into<String>,
575        V: Into<String>,
576    {
577        Self::SuiteCustom {
578            component: component.into(),
579            filename: filename.into(),
580            contents: contents.into(),
581        }
582    }
583
584    pub fn suite_stopped(status: ftest_manager::SuiteResult) -> Self {
585        Self::SuiteStopped { status }
586    }
587
588    pub fn debug_data<S>(filename: S, socket: fidl::Socket) -> Self
589    where
590        S: Into<String>,
591    {
592        Self::DebugData { filename: filename.into(), socket }
593    }
594
595    /// Returns the name of the test case to which the event belongs, if applicable.
596    pub fn test_case_name(&self) -> Option<&String> {
597        match self {
598            RunEvent::CaseFound { name }
599            | RunEvent::CaseStarted { name }
600            | RunEvent::CaseStdout { name, .. }
601            | RunEvent::CaseStderr { name, .. }
602            | RunEvent::CaseStopped { name, .. }
603            | RunEvent::CaseFinished { name } => Some(name),
604            RunEvent::SuiteStarted
605            | RunEvent::SuiteStopped { .. }
606            | RunEvent::SuiteCustom { .. }
607            | RunEvent::DebugData { .. } => None,
608        }
609    }
610
611    /// Same as `test_case_name`, but returns an owned `Option<String>`.
612    pub fn owned_test_case_name(&self) -> Option<String> {
613        self.test_case_name().map(String::from)
614    }
615}
616
617/// Groups events by stdout, stderr and non stdout/stderr events to make it easy to compare them
618/// in tests.
619#[derive(Default, Debug, Eq, PartialEq)]
620pub struct GroupedRunEvents {
621    // order of events is maintained.
622    pub non_artifact_events: Vec<RunEvent>,
623    // order of stdout events is maintained.
624    pub stdout_events: Vec<RunEvent>,
625    // order of stderr events is maintained.
626    pub stderr_events: Vec<RunEvent>,
627}
628
629/// Trait allowing iterators over `RunEvent` to be partitioned by test case name.
630pub trait GroupRunEventByTestCase: Iterator<Item = RunEvent> + Sized {
631    /// Groups the `RunEvent`s by test case name into a map that preserves insertion order of
632    /// various types of events.
633    /// The overall order of test cases (by first event) and the orders of events within each test
634    /// case are preserved, but events from different test cases are effectively de-interleaved.
635    ///
636    /// Example:
637    /// ```rust
638    /// use test_diagnostics::{RunEvent, GroupRunEventByTestCase as _};
639    /// use linked_hash_map::LinkedHashMap;
640    ///
641    /// let events: Vec<RunEvent> = get_events();
642    /// let grouped: LinkedHashMap<Option<String>, GroupedRunEvents> =
643    ///     events.into_iter().group_by_test_case_ordered();
644    /// ```
645    fn group_by_test_case_ordered(self) -> LinkedHashMap<Option<String>, GroupedRunEvents> {
646        let mut map = LinkedHashMap::new();
647        for run_event in self {
648            match run_event {
649                RunEvent::CaseStderr { .. } => map
650                    .entry(run_event.owned_test_case_name())
651                    .or_insert_with(|| GroupedRunEvents::default())
652                    .stderr_events
653                    .push(run_event),
654
655                RunEvent::CaseStdout { .. } => map
656                    .entry(run_event.owned_test_case_name())
657                    .or_insert_with(|| GroupedRunEvents::default())
658                    .stdout_events
659                    .push(run_event),
660
661                _ => map
662                    .entry(run_event.owned_test_case_name())
663                    .or_insert_with(|| GroupedRunEvents::default())
664                    .non_artifact_events
665                    .push(run_event),
666            }
667        }
668        map
669    }
670
671    /// Groups the `RunEvent`s by test case name into an unordered map. The orders of events within
672    /// each test case are preserved, but the test cases themselves are not in a defined order.
673    fn group_by_test_case_unordered(self) -> HashMap<Option<String>, GroupedRunEvents> {
674        let mut map = HashMap::new();
675        for run_event in self {
676            match run_event {
677                RunEvent::CaseStderr { .. } => map
678                    .entry(run_event.owned_test_case_name())
679                    .or_insert_with(|| GroupedRunEvents::default())
680                    .stderr_events
681                    .push(run_event),
682
683                RunEvent::CaseStdout { .. } => map
684                    .entry(run_event.owned_test_case_name())
685                    .or_insert_with(|| GroupedRunEvents::default())
686                    .stdout_events
687                    .push(run_event),
688
689                _ => map
690                    .entry(run_event.owned_test_case_name())
691                    .or_insert_with(|| GroupedRunEvents::default())
692                    .non_artifact_events
693                    .push(run_event),
694            }
695        }
696        map
697    }
698
699    /// Group `RunEvent`s by stdout, stderr and non-stdout/err events and returns `GroupedRunEvents`.
700    fn group(self) -> GroupedRunEvents {
701        let mut events = GroupedRunEvents::default();
702        for run_event in self {
703            match run_event {
704                RunEvent::CaseStderr { .. } => events.stderr_events.push(run_event),
705
706                RunEvent::CaseStdout { .. } => events.stdout_events.push(run_event),
707
708                _ => events.non_artifact_events.push(run_event),
709            }
710        }
711        events
712    }
713}
714
715impl<T> GroupRunEventByTestCase for T where T: Iterator<Item = RunEvent> + Sized {}
716
717#[derive(Default)]
718struct FidlSuiteEventProcessor {
719    case_map: HashMap<u32, String>,
720    std_output_map: HashMap<u32, Vec<fasync::Task<Result<(), Error>>>>,
721}
722
723impl FidlSuiteEventProcessor {
724    fn new() -> Self {
725        FidlSuiteEventProcessor::default()
726    }
727
728    fn get_test_case_name(&self, identifier: u32) -> String {
729        self.case_map
730            .get(&identifier)
731            .unwrap_or_else(|| panic!("invalid test case identifier: {:?}", identifier))
732            .clone()
733    }
734
735    async fn process(
736        &mut self,
737        event: FidlSuiteEvent,
738        mut sender: mpsc::Sender<SuiteEvent>,
739    ) -> Result<(), Error> {
740        let timestamp = event.timestamp;
741        let e = match event.payload.expect("Details cannot be null, please file bug.") {
742            FidlSuiteEventPayload::CaseFound(cf) => {
743                self.case_map.insert(cf.identifier, cf.test_case_name.clone());
744                SuiteEvent::case_found(timestamp, cf.test_case_name).into()
745            }
746            FidlSuiteEventPayload::CaseStarted(cs) => {
747                let test_case_name = self.get_test_case_name(cs.identifier);
748                SuiteEvent::case_started(timestamp, test_case_name).into()
749            }
750            FidlSuiteEventPayload::CaseStopped(cs) => {
751                let test_case_name = self.get_test_case_name(cs.identifier);
752                if let Some(outputs) = self.std_output_map.remove(&cs.identifier) {
753                    for s in outputs {
754                        s.await.context(format!(
755                            "error collecting stdout/stderr of {}",
756                            test_case_name
757                        ))?;
758                    }
759                }
760                SuiteEvent::case_stopped(timestamp, test_case_name, to_case_result(cs.status))
761                    .into()
762            }
763            FidlSuiteEventPayload::CaseFinished(cf) => {
764                let test_case_name = self.get_test_case_name(cf.identifier);
765                SuiteEvent::case_finished(timestamp, test_case_name).into()
766            }
767            FidlSuiteEventPayload::CaseArtifact(ca) => {
768                let name = self.get_test_case_name(ca.identifier);
769                match ca.artifact {
770                    ftest_manager::Artifact::Stdout(stdout) => {
771                        let (s, mut r) = mpsc::channel(1024);
772                        let stdout_task =
773                            fasync::Task::spawn(collect_and_send_string_output(stdout, s));
774                        let mut sender_clone = sender.clone();
775                        let send_stdout_task = fasync::Task::spawn(async move {
776                            while let Some(msg) = r.next().await {
777                                sender_clone
778                                    .send(SuiteEvent::case_stdout(None, &name, msg))
779                                    .await
780                                    .context(format!("cannot send logs for {}", name))?;
781                            }
782                            Ok(())
783                        });
784                        match self.std_output_map.get_mut(&ca.identifier) {
785                            Some(v) => {
786                                v.push(stdout_task);
787                                v.push(send_stdout_task);
788                            }
789                            None => {
790                                self.std_output_map
791                                    .insert(ca.identifier, vec![stdout_task, send_stdout_task]);
792                            }
793                        }
794                        None
795                    }
796                    ftest_manager::Artifact::Stderr(stderr) => {
797                        let (s, mut r) = mpsc::channel(1024);
798                        let stderr_task =
799                            fasync::Task::spawn(collect_and_send_string_output(stderr, s));
800                        let mut sender_clone = sender.clone();
801                        let send_stderr_task = fasync::Task::spawn(async move {
802                            while let Some(msg) = r.next().await {
803                                sender_clone
804                                    .send(SuiteEvent::case_stderr(None, &name, msg))
805                                    .await
806                                    .context(format!("cannot send logs for {}", name))?;
807                            }
808                            Ok(())
809                        });
810                        match self.std_output_map.get_mut(&ca.identifier) {
811                            Some(v) => {
812                                v.push(stderr_task);
813                                v.push(send_stderr_task);
814                            }
815                            None => {
816                                self.std_output_map
817                                    .insert(ca.identifier, vec![stderr_task, send_stderr_task]);
818                            }
819                        }
820                        None
821                    }
822                    ftest_manager::Artifact::Log(log) => match LogStream::from_syslog(log) {
823                        Ok(log_stream) => {
824                            SuiteEvent::test_case_log(timestamp, name, log_stream).into()
825                        }
826                        Err(e) => {
827                            warn!("Cannot collect logs for test suite: {:?}", e);
828                            None
829                        }
830                    },
831                    _ => {
832                        panic!("not supported")
833                    }
834                }
835            }
836            FidlSuiteEventPayload::SuiteArtifact(sa) => match sa.artifact {
837                ftest_manager::Artifact::Stdout(_) => {
838                    panic!("not supported")
839                }
840                ftest_manager::Artifact::Stderr(_) => {
841                    panic!("not supported")
842                }
843                ftest_manager::Artifact::Log(log) => match LogStream::from_syslog(log) {
844                    Ok(log_stream) => SuiteEvent::suite_log(timestamp, log_stream).into(),
845                    Err(e) => {
846                        warn!("Cannot collect logs for test suite: {:?}", e);
847                        None
848                    }
849                },
850                ftest_manager::Artifact::Custom(custom_artifact) => {
851                    let ftest_manager::DirectoryAndToken { directory, token } =
852                        custom_artifact.directory_and_token.unwrap();
853                    let component_moniker = custom_artifact.component_moniker.unwrap();
854                    let mut sender_clone = sender.clone();
855                    fasync::Task::spawn(async move {
856                        let directory = directory.into_proxy();
857                        let entries: Vec<_> =
858                            fuchsia_fs::directory::readdir_recursive(&directory, None)
859                                .try_collect()
860                                .await
861                                .expect("read custom artifact directory");
862                        for entry in entries.into_iter() {
863                            let file = fuchsia_fs::directory::open_file_async(
864                                &directory,
865                                &entry.name,
866                                fio::PERM_READABLE,
867                            )
868                            .unwrap();
869                            let contents = fuchsia_fs::file::read_to_string(&file).await.unwrap();
870                            sender_clone
871                                .send(SuiteEvent::suite_custom(
872                                    timestamp,
873                                    component_moniker.clone(),
874                                    entry.name,
875                                    contents,
876                                ))
877                                .await
878                                .unwrap();
879                        }
880                        // Drop the token here - we must keep the token open for the duration that
881                        // the directory is in use.
882                        drop(token);
883                    })
884                    .detach();
885                    None
886                }
887                _ => {
888                    panic!("not supported")
889                }
890            },
891            FidlSuiteEventPayload::SuiteStarted(_started) => SuiteEvent {
892                timestamp,
893                payload: SuiteEventPayload::RunEvent(RunEvent::SuiteStarted),
894            }
895            .into(),
896            FidlSuiteEventPayload::SuiteStopped(stopped) => SuiteEvent {
897                timestamp,
898                payload: SuiteEventPayload::RunEvent(RunEvent::SuiteStopped {
899                    status: to_suite_result(stopped.status),
900                }),
901            }
902            .into(),
903            SuiteEventPayloadUnknown!() => panic!("Unrecognized SuiteEvent"),
904        };
905        if let Some(item) = e {
906            sender.send(item).await.context("Cannot send event")?;
907        }
908        Ok(())
909    }
910
911    async fn process_event(
912        &mut self,
913        event: ftest_manager::Event,
914        mut sender: mpsc::Sender<SuiteEvent>,
915        filter_debug_data: bool,
916        compressed_debug_data: bool,
917    ) -> Result<(), Error> {
918        let timestamp = event.timestamp;
919        let e = match event.details.expect("Details cannot be null, please file bug.") {
920            ftest_manager::EventDetails::TestCaseFound(cf) => {
921                let test_case_name =
922                    cf.test_case_name.expect("test_case_name must be specified, please file bug.");
923                self.case_map.insert(
924                    cf.test_case_id.expect("test_case_id must be specified, please file bug."),
925                    test_case_name.clone(),
926                );
927                SuiteEvent::case_found(timestamp, test_case_name).into()
928            }
929            ftest_manager::EventDetails::TestCaseStarted(cs) => {
930                let test_case_name = self.get_test_case_name(
931                    cs.test_case_id.expect("test_case_id must be specified, please file bug."),
932                );
933                SuiteEvent::case_started(timestamp, test_case_name).into()
934            }
935            ftest_manager::EventDetails::TestCaseStopped(cs) => {
936                let test_case_name = self.get_test_case_name(
937                    cs.test_case_id.expect("test_case_id must be specified, please file bug."),
938                );
939                if let Some(outputs) = self.std_output_map.remove(
940                    &cs.test_case_id.expect("test_case_id must be specified, please file bug."),
941                ) {
942                    for s in outputs {
943                        s.await.context(format!(
944                            "error collecting stdout/stderr of {}",
945                            test_case_name
946                        ))?;
947                    }
948                }
949                SuiteEvent::case_stopped(
950                    timestamp,
951                    test_case_name,
952                    cs.result.expect("result must be specified, please file bug."),
953                )
954                .into()
955            }
956            ftest_manager::EventDetails::TestCaseFinished(cf) => {
957                let test_case_name = self.get_test_case_name(
958                    cf.test_case_id.expect("test_case_id must be specified, please file bug."),
959                );
960                SuiteEvent::case_finished(timestamp, test_case_name).into()
961            }
962            ftest_manager::EventDetails::TestCaseArtifactGenerated(ca) => {
963                let name = self.get_test_case_name(
964                    ca.test_case_id.expect("test_case_id must be specified, please file bug."),
965                );
966                match ca.artifact.expect("artifact must be specified, please file bug.") {
967                    ftest_manager::Artifact::Stdout(stdout) => {
968                        let (s, mut r) = mpsc::channel(1024);
969                        let stdout_task =
970                            fasync::Task::spawn(collect_and_send_string_output(stdout, s));
971                        let mut sender_clone = sender.clone();
972                        let send_stdout_task = fasync::Task::spawn(async move {
973                            while let Some(msg) = r.next().await {
974                                sender_clone
975                                    .send(SuiteEvent::case_stdout(None, &name, msg))
976                                    .await
977                                    .context(format!("cannot send logs for {}", name))?;
978                            }
979                            Ok(())
980                        });
981                        match self.std_output_map.get_mut(
982                            &ca.test_case_id
983                                .expect("test_case_id must be specified, please file bug."),
984                        ) {
985                            Some(v) => {
986                                v.push(stdout_task);
987                                v.push(send_stdout_task);
988                            }
989                            None => {
990                                self.std_output_map.insert(
991                                    ca.test_case_id
992                                        .expect("test_case_id must be specified, please file bug."),
993                                    vec![stdout_task, send_stdout_task],
994                                );
995                            }
996                        }
997                        None
998                    }
999                    ftest_manager::Artifact::Stderr(stderr) => {
1000                        let (s, mut r) = mpsc::channel(1024);
1001                        let stderr_task =
1002                            fasync::Task::spawn(collect_and_send_string_output(stderr, s));
1003                        let mut sender_clone = sender.clone();
1004                        let send_stderr_task = fasync::Task::spawn(async move {
1005                            while let Some(msg) = r.next().await {
1006                                sender_clone
1007                                    .send(SuiteEvent::case_stderr(None, &name, msg))
1008                                    .await
1009                                    .context(format!("cannot send logs for {}", name))?;
1010                            }
1011                            Ok(())
1012                        });
1013                        match self.std_output_map.get_mut(
1014                            &ca.test_case_id
1015                                .expect("test_case_id must be specified, please file bug."),
1016                        ) {
1017                            Some(v) => {
1018                                v.push(stderr_task);
1019                                v.push(send_stderr_task);
1020                            }
1021                            None => {
1022                                self.std_output_map.insert(
1023                                    ca.test_case_id
1024                                        .expect("test_case_id must be specified, please file bug."),
1025                                    vec![stderr_task, send_stderr_task],
1026                                );
1027                            }
1028                        }
1029                        None
1030                    }
1031                    ftest_manager::Artifact::Log(log) => match LogStream::from_syslog(log) {
1032                        Ok(log_stream) => {
1033                            SuiteEvent::test_case_log(timestamp, name, log_stream).into()
1034                        }
1035                        Err(e) => {
1036                            warn!("Cannot collect logs for test suite: {:?}", e);
1037                            None
1038                        }
1039                    },
1040                    _ => {
1041                        panic!("not supported")
1042                    }
1043                }
1044            }
1045            ftest_manager::EventDetails::SuiteArtifactGenerated(sa) => {
1046                match sa.artifact.expect("artifact must be specified, please file bug.") {
1047                    ftest_manager::Artifact::Stdout(_) => {
1048                        panic!("not supported")
1049                    }
1050                    ftest_manager::Artifact::Stderr(_) => {
1051                        panic!("not supported")
1052                    }
1053                    ftest_manager::Artifact::Log(log) => match LogStream::from_syslog(log) {
1054                        Ok(log_stream) => SuiteEvent::suite_log(timestamp, log_stream).into(),
1055                        Err(e) => {
1056                            warn!("Cannot collect logs for test suite: {:?}", e);
1057                            None
1058                        }
1059                    },
1060                    ftest_manager::Artifact::Custom(custom_artifact) => {
1061                        let ftest_manager::DirectoryAndToken { directory, token } =
1062                            custom_artifact.directory_and_token.unwrap();
1063                        let component_moniker = custom_artifact.component_moniker.unwrap();
1064                        let mut sender_clone = sender.clone();
1065                        fasync::Task::spawn(async move {
1066                            let directory = directory.into_proxy();
1067                            let entries: Vec<_> =
1068                                fuchsia_fs::directory::readdir_recursive(&directory, None)
1069                                    .try_collect()
1070                                    .await
1071                                    .expect("read custom artifact directory");
1072                            for entry in entries.into_iter() {
1073                                let file = fuchsia_fs::directory::open_file_async(
1074                                    &directory,
1075                                    &entry.name,
1076                                    fio::PERM_READABLE,
1077                                )
1078                                .unwrap();
1079                                let contents =
1080                                    fuchsia_fs::file::read_to_string(&file).await.unwrap();
1081                                sender_clone
1082                                    .send(SuiteEvent::suite_custom(
1083                                        timestamp,
1084                                        component_moniker.clone(),
1085                                        entry.name,
1086                                        contents,
1087                                    ))
1088                                    .await
1089                                    .unwrap();
1090                            }
1091                            // Drop the token here - we must keep the token open for the duration that
1092                            // the directory is in use.
1093                            drop(token);
1094                        })
1095                        .detach();
1096                        None
1097                    }
1098                    ftest_manager::Artifact::DebugData(iterator) => {
1099                        if !filter_debug_data {
1100                            let mut sender_clone = sender.clone();
1101                            let proxy = iterator.into_proxy();
1102                            fasync::Task::spawn(async move {
1103                                loop {
1104                                    let data = match compressed_debug_data {
1105                                        true => proxy.get_next_compressed().await.unwrap(),
1106                                        false => proxy.get_next().await.unwrap(),
1107                                    };
1108                                    if data.is_empty() {
1109                                        break;
1110                                    }
1111                                    for data_file in data {
1112                                        let socket =
1113                                            data_file.socket.expect("File cannot be empty");
1114                                        sender_clone
1115                                            .send(SuiteEvent::debug_data(
1116                                                timestamp,
1117                                                data_file.name.expect("Name cannot be empty"),
1118                                                socket,
1119                                            ))
1120                                            .await
1121                                            .unwrap();
1122                                    }
1123                                }
1124                            })
1125                            .detach();
1126                        }
1127                        None
1128                    }
1129                    _ => {
1130                        panic!("not supported")
1131                    }
1132                }
1133            }
1134            ftest_manager::EventDetails::SuiteStarted(_started) => SuiteEvent {
1135                timestamp,
1136                payload: SuiteEventPayload::RunEvent(RunEvent::SuiteStarted),
1137            }
1138            .into(),
1139            ftest_manager::EventDetails::SuiteStopped(stopped) => SuiteEvent {
1140                timestamp,
1141                payload: SuiteEventPayload::RunEvent(RunEvent::SuiteStopped {
1142                    status: stopped.result.expect("result must be specified, please file bug."),
1143                }),
1144            }
1145            .into(),
1146            EventDetailsUnknown!() => panic!("Unrecognized EventDetails"),
1147        };
1148        if let Some(item) = e {
1149            sender.send(item).await.context("Cannot send event")?;
1150        }
1151        Ok(())
1152    }
1153}
1154
1155#[derive(Debug, thiserror::Error, Eq, PartialEq, Copy, Clone)]
1156pub enum SuiteLaunchError {
1157    #[error("Cannot enumerate tests")]
1158    CaseEnumeration,
1159
1160    #[error("Cannot resolve test url")]
1161    InstanceCannotResolve,
1162
1163    #[error("Invalid arguments passed")]
1164    InvalidArgs,
1165
1166    #[error("Cannot connect to test suite")]
1167    FailedToConnectToTestSuite,
1168
1169    #[error("resource unavailable")]
1170    ResourceUnavailable,
1171
1172    #[error("Some internal error occurred. Please file bug")]
1173    InternalError,
1174
1175    #[error("No test cases matched the provided filters")]
1176    NoMatchingCases,
1177}
1178
1179impl From<ftest_manager::LaunchError> for SuiteLaunchError {
1180    fn from(error: ftest_manager::LaunchError) -> Self {
1181        match error {
1182            ftest_manager::LaunchError::ResourceUnavailable => {
1183                SuiteLaunchError::ResourceUnavailable
1184            }
1185            ftest_manager::LaunchError::InstanceCannotResolve => {
1186                SuiteLaunchError::InstanceCannotResolve
1187            }
1188            ftest_manager::LaunchError::InvalidArgs => SuiteLaunchError::InvalidArgs,
1189            ftest_manager::LaunchError::FailedToConnectToTestSuite => {
1190                SuiteLaunchError::FailedToConnectToTestSuite
1191            }
1192            ftest_manager::LaunchError::CaseEnumeration => SuiteLaunchError::CaseEnumeration,
1193            ftest_manager::LaunchError::InternalError => SuiteLaunchError::InternalError,
1194            ftest_manager::LaunchError::NoMatchingCases => SuiteLaunchError::NoMatchingCases,
1195            ftest_manager::LaunchErrorUnknown!() => panic!("Encountered unknown launch error"),
1196        }
1197    }
1198}
1199
1200/// Instance to control a single test suite run.
1201pub struct SuiteRunInstance {
1202    controller_proxy: Arc<SuiteControllerProxy>,
1203}
1204
1205impl SuiteRunInstance {
1206    pub fn controller(&self) -> Arc<SuiteControllerProxy> {
1207        self.controller_proxy.clone()
1208    }
1209
1210    pub async fn collect_events(&self, sender: mpsc::Sender<SuiteEvent>) -> Result<(), Error> {
1211        let controller_proxy = self.controller_proxy.clone();
1212        let mut processor = FidlSuiteEventProcessor::new();
1213        loop {
1214            match controller_proxy.get_events().await? {
1215                Err(e) => return Err(SuiteLaunchError::from(e).into()),
1216                Ok(events) => {
1217                    if events.len() == 0 {
1218                        break;
1219                    }
1220                    for event in events {
1221                        if let Err(e) = processor.process(event, sender.clone()).await {
1222                            warn!("error running test suite: {:?}", e);
1223                            let _ = controller_proxy.kill();
1224                            return Ok(());
1225                        }
1226                    }
1227                }
1228            }
1229        }
1230        Ok(())
1231    }
1232
1233    pub async fn collect_events_with_watch(
1234        &self,
1235        sender: mpsc::Sender<SuiteEvent>,
1236        filter_debug_data: bool,
1237        compressed_debug_data: bool,
1238    ) -> Result<(), Error> {
1239        let controller_proxy = self.controller_proxy.clone();
1240        let mut processor = FidlSuiteEventProcessor::new();
1241        loop {
1242            match controller_proxy.watch_events().await? {
1243                Err(e) => return Err(SuiteLaunchError::from(e).into()),
1244                Ok(events) => {
1245                    if events.len() == 0 {
1246                        break;
1247                    }
1248                    for event in events {
1249                        if let Err(e) = processor
1250                            .process_event(
1251                                event,
1252                                sender.clone(),
1253                                filter_debug_data,
1254                                compressed_debug_data,
1255                            )
1256                            .await
1257                        {
1258                            warn!("error running test suite: {:?}", e);
1259                            let _ = controller_proxy.kill();
1260                            return Ok(());
1261                        }
1262                    }
1263                }
1264            }
1265        }
1266        Ok(())
1267    }
1268}
1269
1270fn to_case_result(outcome: ftest_manager::CaseStatus) -> ftest_manager::TestCaseResult {
1271    match outcome {
1272        ftest_manager::CaseStatus::Passed => ftest_manager::TestCaseResult::Passed,
1273        ftest_manager::CaseStatus::Failed => ftest_manager::TestCaseResult::Failed,
1274        ftest_manager::CaseStatus::TimedOut => ftest_manager::TestCaseResult::TimedOut,
1275        ftest_manager::CaseStatus::Skipped => ftest_manager::TestCaseResult::Skipped,
1276        ftest_manager::CaseStatus::Error => ftest_manager::TestCaseResult::Error,
1277        _ => ftest_manager::TestCaseResult::Error,
1278    }
1279}
1280
1281fn to_suite_result(outcome: ftest_manager::SuiteStatus) -> ftest_manager::SuiteResult {
1282    match outcome {
1283        ftest_manager::SuiteStatus::Passed => ftest_manager::SuiteResult::Finished,
1284        ftest_manager::SuiteStatus::Failed => ftest_manager::SuiteResult::Failed,
1285        ftest_manager::SuiteStatus::DidNotFinish => ftest_manager::SuiteResult::DidNotFinish,
1286        ftest_manager::SuiteStatus::TimedOut => ftest_manager::SuiteResult::TimedOut,
1287        ftest_manager::SuiteStatus::Stopped => ftest_manager::SuiteResult::Stopped,
1288        ftest_manager::SuiteStatus::InternalError => ftest_manager::SuiteResult::InternalError,
1289        _ => ftest_manager::SuiteResult::InternalError,
1290    }
1291}
test_manager_test_lib/test_lib.rs

test_manager_test_lib/
test_lib.rs
