test_manager_lib/
running_suite.rs

1// Copyright 2022 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::above_root_capabilities::AboveRootCapabilitiesForTest;
6use crate::constants::{
7    CUSTOM_ARTIFACTS_CAPABILITY_NAME, HERMETIC_RESOLVER_REALM_NAME, TEST_ENVIRONMENT_NAME,
8    TEST_ROOT_COLLECTION, TEST_ROOT_REALM_NAME, WRAPPER_REALM_NAME,
9};
10use crate::debug_agent::DebugAgent;
11use crate::debug_data_processor::{serve_debug_data_publisher, DebugDataSender};
12use crate::error::{DebugAgentError, LaunchTestError};
13use crate::offers::apply_offers;
14use crate::run_events::SuiteEvents;
15use crate::self_diagnostics::DiagnosticNode;
16use crate::test_suite::SuiteRealm;
17use crate::utilities::stream_fn;
18use crate::{diagnostics, facet, resolver};
19use anyhow::{anyhow, format_err, Context, Error};
20use fidl::endpoints::{create_proxy, ClientEnd};
21use fidl_fuchsia_component_resolution::ResolverProxy;
22use fidl_fuchsia_pkg::PackageResolverProxy;
23use ftest::Invocation;
24use ftest_manager::{CaseStatus, LaunchError, SuiteStatus};
25use fuchsia_async::{self as fasync, TimeoutExt};
26use fuchsia_component::client::connect_to_protocol_at_dir_root;
27use fuchsia_component_test::error::Error as RealmBuilderError;
28use fuchsia_component_test::{
29    Capability, ChildOptions, RealmBuilder, RealmBuilderParams, RealmInstance, Ref, Route,
30};
31use fuchsia_url::AbsoluteComponentUrl;
32use futures::channel::{mpsc, oneshot};
33use futures::future::join_all;
34use futures::prelude::*;
35use futures::{lock, FutureExt};
36use log::{debug, error, info, warn};
37use moniker::Moniker;
38use resolver::AllowedPackages;
39use std::collections::HashSet;
40use std::fmt;
41use std::sync::atomic::{AtomicU32, Ordering};
42use std::sync::Arc;
43use thiserror::Error;
44use {
45    fidl_fuchsia_component as fcomponent, fidl_fuchsia_component_decl as fdecl,
46    fidl_fuchsia_diagnostics as fdiagnostics, fidl_fuchsia_io as fio, fidl_fuchsia_sys2 as fsys,
47    fidl_fuchsia_test as ftest, fidl_fuchsia_test_manager as ftest_manager,
48};
49
50const DEBUG_DATA_REALM_NAME: &'static str = "debug-data";
51const ARCHIVIST_REALM_NAME: &'static str = "archivist";
52const DIAGNOSTICS_DICTIONARY_NAME: &'static str = "diagnostics";
53const ARCHIVIST_FOR_EMBEDDING_URL: &'static str = "#meta/archivist-for-embedding.cm";
54const MEMFS_FOR_EMBEDDING_URL: &'static str = "#meta/memfs.cm";
55const MEMFS_REALM_NAME: &'static str = "memfs";
56
57pub const HERMETIC_RESOLVER_CAPABILITY_NAME: &'static str = "hermetic_resolver";
58
59/// A |RunningSuite| represents a launched test component.
60pub(crate) struct RunningSuite {
61    instance: RealmInstance,
62    mock_ready_task: Option<fasync::Task<()>>,
63    logs_iterator_task: Option<fasync::Task<Result<(), Error>>>,
64    /// A connection to the embedded archivist LogSettings protocol. This connection must be kept
65    /// alive for the duration of the test execution for the configured settings to be persisted.
66    log_settings: Option<fdiagnostics::LogSettingsProxy>,
67    /// Server ends of event pairs used to track if a client is accessing a component's
68    /// custom storage. Used to defer destruction of the realm until clients have completed
69    /// reading the storage.
70    custom_artifact_tokens: Vec<zx::EventPair>,
71
72    /// `Realm` protocol for the test root.
73    test_realm_proxy: fcomponent::RealmProxy,
74    /// exposed directory of the test realm.
75    exposed_dir: fio::DirectoryProxy,
76
77    /// Wrapper for `DebugAgent` service.
78    debug_agent: Option<DebugAgent>,
79}
80
81impl RunningSuite {
82    /// Launch a suite component.
83    pub(crate) async fn launch(
84        test_url: &str,
85        facets: facet::SuiteFacets,
86        resolver: Arc<ResolverProxy>,
87        pkg_resolver: Arc<PackageResolverProxy>,
88        above_root_capabilities_for_test: Arc<AboveRootCapabilitiesForTest>,
89        debug_data_sender: DebugDataSender,
90        diagnostics: &DiagnosticNode,
91        suite_realm: &Option<SuiteRealm>,
92        use_debug_agent: bool,
93    ) -> Result<Self, LaunchTestError> {
94        info!(test_url, diagnostics:?, collection = facets.collection; "Starting test suite.");
95
96        let test_package = match AbsoluteComponentUrl::parse(test_url) {
97            Ok(component_url) => component_url.package_url().name().to_string(),
98            Err(_) => match fuchsia_url::boot_url::BootUrl::parse(test_url) {
99                Ok(boot_url) => boot_url.path().to_string(),
100                Err(_) => {
101                    return Err(LaunchTestError::InvalidResolverData);
102                }
103            },
104        };
105        above_root_capabilities_for_test
106            .validate(facets.collection)
107            .map_err(LaunchTestError::ValidateTestRealm)?;
108        let (builder, mock_ready_event) = get_realm(
109            test_url,
110            test_package.as_ref(),
111            &facets,
112            above_root_capabilities_for_test,
113            resolver,
114            pkg_resolver,
115            debug_data_sender,
116            suite_realm,
117        )
118        .await
119        .map_err(LaunchTestError::InitializeTestRealm)?;
120        let instance = builder.build().await.map_err(LaunchTestError::CreateTestRealm)?;
121        let test_realm_proxy: fcomponent::RealmProxy = instance
122            .root
123            .connect_to_protocol_at_exposed_dir()
124            .map_err(|e| LaunchTestError::ConnectToTestSuite(e))?;
125        let collection_ref = fdecl::CollectionRef { name: TEST_ROOT_COLLECTION.into() };
126        let child_decl = fdecl::Child {
127            name: Some(TEST_ROOT_REALM_NAME.into()),
128            url: Some(test_url.into()),
129            startup: Some(fdecl::StartupMode::Eager),
130            environment: None,
131            ..Default::default()
132        };
133        test_realm_proxy
134            .create_child(&collection_ref, &child_decl, fcomponent::CreateChildArgs::default())
135            .await
136            .map_err(|e| LaunchTestError::CreateTestFidl(e))?
137            .map_err(|e| LaunchTestError::CreateTest(e))?;
138
139        let (exposed_dir, server_end) = fidl::endpoints::create_proxy::<fio::DirectoryMarker>();
140        let child_ref = fdecl::ChildRef {
141            name: TEST_ROOT_REALM_NAME.into(),
142            collection: Some(TEST_ROOT_COLLECTION.into()),
143        };
144        test_realm_proxy
145            .open_exposed_dir(&child_ref, server_end)
146            .await
147            .map_err(|e| LaunchTestError::ConnectToTestSuite(e.into()))?
148            .map_err(|e| {
149                LaunchTestError::ConnectToTestSuite(format_err!(
150                    "failed to open exposed dir: {:?}",
151                    e
152                ))
153            })?;
154
155        Ok(RunningSuite {
156            custom_artifact_tokens: vec![],
157            mock_ready_task: Some(fasync::Task::spawn(
158                mock_ready_event.wait_or_dropped().map(|_| ()),
159            )),
160            logs_iterator_task: None,
161            log_settings: None,
162            instance,
163            test_realm_proxy,
164            exposed_dir,
165            debug_agent: if use_debug_agent {
166                DebugAgent::new().await.map_err(log_warning).ok()
167            } else {
168                warn!("Not using debug_agent for test {} because it is marked create_no_exception_channel", test_url);
169                None
170            },
171        })
172    }
173
174    pub(crate) async fn run_tests(
175        &mut self,
176        test_url: &str,
177        mut options: ftest_manager::RunOptions,
178        mut sender: mpsc::Sender<Result<SuiteEvents, LaunchError>>,
179        mut stop_recv: oneshot::Receiver<()>,
180    ) {
181        debug!("running test suite {}", test_url);
182
183        let (log_iterator, syslog) = match options.log_iterator {
184            Some(ftest_manager::LogsIteratorOption::SocketBatchIterator) => {
185                let (local, remote) = zx::Socket::create_stream();
186                (ftest_manager::LogsIterator::Stream(local), ftest_manager::Syslog::Stream(remote))
187            }
188            _ => {
189                let (proxy, request) = fidl::endpoints::create_endpoints();
190                (ftest_manager::LogsIterator::Batch(request), ftest_manager::Syslog::Batch(proxy))
191            }
192        };
193
194        sender.send(Ok(SuiteEvents::suite_syslog(syslog).into())).await.unwrap();
195
196        if let Some(log_interest) = options.log_interest.take() {
197            let log_settings: fdiagnostics::LogSettingsProxy =
198                match self.instance.root.connect_to_protocol_at_exposed_dir() {
199                    Ok(proxy) => proxy,
200                    Err(e) => {
201                        warn!("Error connecting to LogSettings");
202                        sender
203                            .send(Err(LaunchTestError::ConnectToLogSettings(e.into()).into()))
204                            .await
205                            .unwrap();
206                        return;
207                    }
208                };
209
210            let fut = log_settings.set_interest(&log_interest);
211            if let Err(e) = fut.await {
212                warn!("Error setting log interest");
213                sender.send(Err(LaunchTestError::SetLogInterest(e.into()).into())).await.unwrap();
214                return;
215            }
216            self.log_settings = Some(log_settings);
217        }
218
219        let archive_accessor = match self
220            .instance
221            .root
222            .connect_to_protocol_at_exposed_dir::<fdiagnostics::ArchiveAccessorProxy>()
223        {
224            Ok(accessor) => match accessor.wait_for_ready().await {
225                Ok(()) => accessor,
226                Err(e) => {
227                    warn!("Error connecting to ArchiveAccessor");
228                    sender
229                        .send(Err(LaunchTestError::ConnectToArchiveAccessor(e.into()).into()))
230                        .await
231                        .unwrap();
232                    return;
233                }
234            },
235            Err(e) => {
236                warn!("Error connecting to ArchiveAccessor");
237                sender
238                    .send(Err(LaunchTestError::ConnectToArchiveAccessor(e.into()).into()))
239                    .await
240                    .unwrap();
241                return;
242            }
243        };
244        let host_archive_accessor = match self.instance.root.connect_to_protocol_at_exposed_dir() {
245            Ok(accessor) => accessor,
246            Err(e) => {
247                warn!("Error connecting to ArchiveAccessor");
248                sender
249                    .send(Err(LaunchTestError::ConnectToArchiveAccessor(e.into()).into()))
250                    .await
251                    .unwrap();
252                return;
253            }
254        };
255
256        match diagnostics::serve_syslog(archive_accessor, host_archive_accessor, log_iterator) {
257            Ok(diagnostics::ServeSyslogOutcome { logs_iterator_task }) => {
258                self.logs_iterator_task = logs_iterator_task;
259            }
260            Err(e) => {
261                warn!("Error spawning iterator server: {:?}", e);
262                sender.send(Err(LaunchTestError::StreamIsolatedLogs(e).into())).await.unwrap();
263                return;
264            }
265        };
266
267        let fut = async {
268            let matcher = match options.case_filters_to_run.as_ref() {
269                Some(filters) => match CaseMatcher::new(filters) {
270                    Ok(p) => Some(p),
271                    Err(e) => {
272                        sender.send(Err(LaunchError::InvalidArgs)).await.unwrap();
273                        return Err(e);
274                    }
275                },
276                None => None,
277            };
278
279            let suite = self.connect_to_suite()?;
280            let invocations = match enumerate_test_cases(&suite, matcher.as_ref()).await {
281                Ok(i) if i.is_empty() && matcher.is_some() => {
282                    sender.send(Err(LaunchError::NoMatchingCases)).await.unwrap();
283                    return Err(format_err!("Found no matching cases using {:?}", matcher));
284                }
285                Ok(i) => i,
286                Err(e) => {
287                    sender.send(Err(LaunchError::CaseEnumeration)).await.unwrap();
288                    return Err(e);
289                }
290            };
291            if let Ok(Some(_)) = stop_recv.try_recv() {
292                sender
293                    .send(Ok(SuiteEvents::suite_stopped(SuiteStatus::Stopped).into()))
294                    .await
295                    .unwrap();
296                return Ok(());
297            }
298
299            let mut suite_status = SuiteStatus::Passed;
300            let mut invocations_iter = invocations.into_iter();
301            let counter = AtomicU32::new(0);
302            let timeout_time = match options.timeout {
303                Some(t) => zx::MonotonicInstant::after(zx::MonotonicDuration::from_nanos(t)),
304                None => zx::MonotonicInstant::INFINITE,
305            };
306            let timeout_fut = fasync::Timer::new(timeout_time).shared();
307
308            let run_options = get_invocation_options(options);
309
310            sender.send(Ok(SuiteEvents::suite_started().into())).await.unwrap();
311
312            loop {
313                const INVOCATIONS_CHUNK: usize = 50;
314                let chunk = invocations_iter.by_ref().take(INVOCATIONS_CHUNK).collect::<Vec<_>>();
315                if chunk.is_empty() {
316                    break;
317                }
318                if let Ok(Some(_)) = stop_recv.try_recv() {
319                    sender
320                        .send(Ok(SuiteEvents::suite_stopped(SuiteStatus::Stopped).into()))
321                        .await
322                        .unwrap();
323                    return self.report_custom_artifacts(&mut sender).await;
324                }
325                let res = match run_invocations(
326                    &suite,
327                    chunk,
328                    run_options.clone(),
329                    &counter,
330                    &mut sender,
331                    timeout_fut.clone(),
332                )
333                .await
334                .context("Error running test cases")
335                {
336                    Ok(success) => success,
337                    Err(e) => {
338                        return Err(e);
339                    }
340                };
341                if res == SuiteStatus::TimedOut {
342                    if let Some(debug_agent) = &self.debug_agent {
343                        debug_agent.report_all_backtraces(sender.clone()).await;
344                    }
345                    sender
346                        .send(Ok(SuiteEvents::suite_stopped(SuiteStatus::TimedOut).into()))
347                        .await
348                        .unwrap();
349                    return self.report_custom_artifacts(&mut sender).await;
350                }
351                suite_status = concat_suite_status(suite_status, res);
352            }
353            sender.send(Ok(SuiteEvents::suite_stopped(suite_status).into())).await.unwrap();
354            if let Some(debug_agent) = &mut self.debug_agent {
355                let mut fatal_exception_receiver = debug_agent.take_fatal_exception_receiver();
356                debug_agent.stop_sending_fatal_exceptions();
357                loop {
358                    match fatal_exception_receiver.next().await {
359                        Some(backtrace_info) => {
360                            let (client_end, mut server_end) =
361                                fidl::handle::fuchsia_handles::Socket::create_stream();
362                            sender
363                                .send(Ok(SuiteEvents::suite_stderr(client_end).into()))
364                                .await
365                                .unwrap();
366
367                            warn!("fatal exception occurred, thread {:?}", backtrace_info.thread);
368                            let _ = server_end.write(
369                                format!("fatal exception, thread {:?}\n", backtrace_info.thread)
370                                    .as_bytes(),
371                            );
372                            DebugAgent::write_backtrace_info(&backtrace_info, &mut server_end)
373                                .await;
374                        }
375                        None => {
376                            break;
377                        }
378                    }
379                }
380            }
381            self.report_custom_artifacts(&mut sender).await
382        };
383        if let Err(e) = fut.await {
384            warn!("Error running test {}: {:?}", test_url, e);
385        }
386    }
387
388    /// Find any custom artifact users under the test realm and report them via sender.
389    async fn report_custom_artifacts(
390        &mut self,
391        sender: &mut mpsc::Sender<Result<SuiteEvents, LaunchError>>,
392    ) -> Result<(), Error> {
393        // TODO(https://fxbug.dev/42074399): Support custom artifacts when test is running in outside
394        // realm.
395        let artifact_storage_admin = self.connect_to_storage_admin()?;
396
397        let root_moniker = "./";
398        let (iterator, iter_server) = create_proxy::<fsys::StorageIteratorMarker>();
399        artifact_storage_admin
400            .list_storage_in_realm(&root_moniker, iter_server)
401            .await?
402            .map_err(|e| format_err!("Error listing storage users in test realm: {:?}", e))?;
403        let stream = stream_fn(move || iterator.next());
404        futures::pin_mut!(stream);
405        while let Some(storage_moniker) = stream.try_next().await? {
406            let (node, server) = fidl::endpoints::create_endpoints::<fio::NodeMarker>();
407            let directory: ClientEnd<fio::DirectoryMarker> = node.into_channel().into();
408            artifact_storage_admin.open_storage(&storage_moniker, server).await?.map_err(|e| {
409                format_err!("Error opening component storage in test realm: {:?}", e)
410            })?;
411            let (event_client, event_server) = zx::EventPair::create();
412            self.custom_artifact_tokens.push(event_server);
413
414            // Monikers should be reported relative to the test root, so strip away the wrapping
415            // components from the path.
416            let moniker_parsed = Moniker::try_from(storage_moniker.as_str()).unwrap();
417            let moniker_relative_to_test_root = if moniker_parsed.is_root() {
418                moniker_parsed
419            } else {
420                Moniker::new_from_borrowed(&moniker_parsed.path()[1..])
421            };
422            sender
423                .send(Ok(SuiteEvents::suite_custom_artifact(ftest_manager::CustomArtifact {
424                    directory_and_token: Some(ftest_manager::DirectoryAndToken {
425                        directory,
426                        token: event_client,
427                    }),
428                    component_moniker: Some(moniker_relative_to_test_root.to_string()),
429                    ..Default::default()
430                })
431                .into()))
432                .await
433                .unwrap();
434        }
435        Ok(())
436    }
437
438    pub(crate) fn connect_to_suite(&self) -> Result<ftest::SuiteProxy, LaunchTestError> {
439        connect_to_protocol_at_dir_root::<ftest::SuiteMarker>(&self.exposed_dir)
440            .map_err(|e| LaunchTestError::ConnectToTestSuite(e))
441    }
442
443    fn connect_to_storage_admin(&self) -> Result<fsys::StorageAdminProxy, LaunchTestError> {
444        self.instance
445            .root
446            .connect_to_protocol_at_exposed_dir()
447            .map_err(|e| LaunchTestError::ConnectToStorageAdmin(e))
448    }
449
450    /// Mark the resources associated with the suite for destruction, then wait for destruction to
451    /// complete. Returns an error only if destruction fails.
452    pub(crate) async fn destroy(self, diagnostics: DiagnosticNode) -> Result<(), Error> {
453        let exposed_dir_fut = self.exposed_dir.close();
454        let exposed_dir_close_task = fasync::Task::spawn(async move {
455            let _ = exposed_dir_fut.await;
456        });
457
458        // before destroying the realm, wait for any clients to finish accessing storage.
459        // TODO(https://fxbug.dev/42165719): Remove signal for USER_0, this is used while Overnet does not support
460        // signalling ZX_EVENTPAIR_CLOSED when the eventpair is closed.
461        let tokens_closed_signals = self.custom_artifact_tokens.iter().map(|token| {
462            fasync::OnSignals::new(token, zx::Signals::EVENTPAIR_PEER_CLOSED | zx::Signals::USER_0)
463                .unwrap_or_else(|_| zx::Signals::empty())
464        });
465        if let Some(mock_ready_task) = self.mock_ready_task {
466            info!(diagnostics:?; "Waiting on mock_ready_task...");
467            mock_ready_task.await;
468        }
469        info!(diagnostics:?; "Waiting on tokens_closed_signals...");
470        futures::future::join_all(tokens_closed_signals).await;
471
472        info!(diagnostics:?; "Waiting on exposed_dir_close_task...");
473        exposed_dir_close_task.await;
474
475        info!(diagnostics:?; "Start destroying test realm...");
476
477        // TODO(https://fxbug.dev/42174479) Remove timeout once component manager hangs are removed.
478        // This value is set to be slightly longer than the shutdown timeout for tests (30 sec).
479        const TEARDOWN_TIMEOUT: zx::MonotonicDuration = zx::MonotonicDuration::from_seconds(32);
480
481        // Make the call to destroy the test, before destroying the entire realm. Once this
482        // completes, it guarantees that any of its service providers (archivist, storage,
483        // debugdata) have received all outgoing requests from the test such as log connections,
484        // etc.
485        let child_ref = fdecl::ChildRef {
486            name: TEST_ROOT_REALM_NAME.into(),
487            collection: Some(TEST_ROOT_COLLECTION.into()),
488        };
489        self.test_realm_proxy
490            .destroy_child(&child_ref)
491            .map_err(|e| Error::from(e).context("call to destroy test failed"))
492            // This should not hang, but wrap it in a timeout just in case.
493            .on_timeout(TEARDOWN_TIMEOUT, || Err(anyhow!("Timeout waiting for test to destroy")))
494            .await?
495            .map_err(|e| format_err!("call to destroy test failed: {:?}", e))?;
496
497        #[derive(Debug, Error)]
498        enum TeardownError {
499            #[error("timeout")]
500            Timeout,
501            #[error("{}", .0)]
502            Other(Error),
503        }
504        // When serving logs over ArchiveIterator in the host, we should also wait for all logs to
505        // be drained.
506        let logs_iterator_task = self
507            .logs_iterator_task
508            .unwrap_or_else(|| fasync::Task::spawn(futures::future::ready(Ok(()))));
509        let (logs_iterator_res, teardown_res) = futures::future::join(
510            logs_iterator_task.map_err(|e| TeardownError::Other(e)).on_timeout(
511                TEARDOWN_TIMEOUT,
512                || {
513                    // This log is detected in triage. Update the config in
514                    // src/diagnostics/config/triage/test_manager.triage when changing this log.
515                    warn!("Test manager timeout draining logs");
516                    Err(TeardownError::Timeout)
517                },
518            ),
519            self.instance.destroy().map_err(|e| TeardownError::Other(e.into())).on_timeout(
520                TEARDOWN_TIMEOUT,
521                || {
522                    // This log is detected in triage. Update the config in
523                    // src/diagnostics/config/triage/test_manager.triage when changing this log.
524                    warn!("Test manager timeout destroying realm");
525                    Err(TeardownError::Timeout)
526                },
527            ),
528        )
529        .await;
530        let logs_iterator_res = match logs_iterator_res {
531            Err(TeardownError::Other(e)) => {
532                // Allow test teardown to proceed if failed to stream logs
533                warn!("Error streaming logs: {}", e);
534                Ok(())
535            }
536            r => r,
537        };
538        match (logs_iterator_res, teardown_res) {
539            (Err(e1), Err(e2)) => {
540                Err(anyhow!("Draining logs failed: {}. Realm teardown failed: {}", e1, e2))
541            }
542            (Err(e), Ok(())) => Err(anyhow!("Draining logs failed: {}", e)),
543            (Ok(()), Err(e)) => Err(anyhow!("Realm teardown failed: {}", e)),
544            (Ok(()), Ok(())) => Ok(()),
545        }
546    }
547}
548
549/// Logs an error an returns it.
550fn log_warning(error: DebugAgentError) -> DebugAgentError {
551    warn!("{}", error);
552    error
553}
554
555/// Enumerates test cases and return invocations.
556pub(crate) async fn enumerate_test_cases(
557    suite: &ftest::SuiteProxy,
558    matcher: Option<&CaseMatcher>,
559) -> Result<Vec<Invocation>, anyhow::Error> {
560    debug!("enumerating tests");
561    let (case_iterator, server_end) = fidl::endpoints::create_proxy();
562    suite.get_tests(server_end).map_err(enumeration_error)?;
563    let mut invocations = vec![];
564
565    loop {
566        let cases = case_iterator.get_next().await.map_err(enumeration_error)?;
567        if cases.is_empty() {
568            break;
569        }
570        for case in cases {
571            let case_name = case.name.ok_or(format_err!("invocation should contain a name."))?;
572            if matcher.as_ref().map_or(true, |m| m.matches(&case_name)) {
573                invocations.push(Invocation {
574                    name: Some(case_name),
575                    tag: None,
576                    ..Default::default()
577                });
578            }
579        }
580    }
581
582    debug!("invocations: {:#?}", invocations);
583
584    Ok(invocations)
585}
586
587pub(crate) struct CaseMatcher {
588    /// Patterns specifying cases to include.
589    includes: Vec<glob::Pattern>,
590    /// Patterns specifying cases to exclude.
591    excludes: Vec<glob::Pattern>,
592}
593
594impl fmt::Debug for CaseMatcher {
595    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
596        f.debug_struct("CaseMatcher")
597            .field("includes", &self.includes.iter().map(|p| p.to_string()).collect::<Vec<_>>())
598            .field("excludes", &self.excludes.iter().map(|p| p.to_string()).collect::<Vec<_>>())
599            .finish()
600    }
601}
602
603impl CaseMatcher {
604    fn new<T: AsRef<str>>(filters: &Vec<T>) -> Result<Self, anyhow::Error> {
605        let mut matcher = CaseMatcher { includes: vec![], excludes: vec![] };
606        filters.iter().try_for_each(|filter| {
607            match filter.as_ref().chars().next() {
608                Some('-') => {
609                    let pattern = glob::Pattern::new(&filter.as_ref()[1..])
610                        .map_err(|e| format_err!("Bad negative test filter pattern: {}", e))?;
611                    matcher.excludes.push(pattern);
612                }
613                Some(_) | None => {
614                    let pattern = glob::Pattern::new(&filter.as_ref())
615                        .map_err(|e| format_err!("Bad test filter pattern: {}", e))?;
616                    matcher.includes.push(pattern);
617                }
618            }
619            Ok::<(), anyhow::Error>(())
620        })?;
621        Ok(matcher)
622    }
623
624    /// Returns whether or not a case should be run.
625    fn matches(&self, case: &str) -> bool {
626        let matches_includes = match self.includes.is_empty() {
627            true => true,
628            false => self.includes.iter().any(|p| p.matches(case)),
629        };
630        matches_includes && !self.excludes.iter().any(|p| p.matches(case))
631    }
632}
633
634fn get_allowed_package_value(suite_facet: &facet::SuiteFacets) -> AllowedPackages {
635    match &suite_facet.deprecated_allowed_packages {
636        Some(deprecated_allowed_packages) => {
637            AllowedPackages::from_iter(deprecated_allowed_packages.iter().cloned())
638        }
639        None => AllowedPackages::zero_allowed_pkgs(),
640    }
641}
642
643/// Create a RealmBuilder and populate it with local components and routes needed to run a
644/// test. Returns a populated RealmBuilder instance, and an Event which is signalled when
645/// the debug_data local component is ready to accept connection requests.
646// TODO(https://fxbug.dev/42056523): The returned event is part of a synchronization mechanism to
647// work around cases when a component is destroyed before starting, even though there is a
648// request. In this case debug data can be lost. Remove the synchronization once it is
649// provided by cm.
650async fn get_realm(
651    test_url: &str,
652    test_package: &str,
653    suite_facet: &facet::SuiteFacets,
654    above_root_capabilities_for_test: Arc<AboveRootCapabilitiesForTest>,
655    resolver: Arc<ResolverProxy>,
656    pkg_resolver: Arc<PackageResolverProxy>,
657    debug_data_sender: DebugDataSender,
658    suite_realm: &Option<SuiteRealm>,
659) -> Result<(RealmBuilder, async_utils::event::Event), RealmBuilderError> {
660    let mut realm_param = RealmBuilderParams::new();
661    realm_param = match suite_realm {
662        Some(suite_realm) => realm_param
663            .with_realm_proxy(suite_realm.realm_proxy.clone())
664            .in_collection(suite_realm.test_collection.clone()),
665        None => realm_param.in_collection(suite_facet.collection.to_string()),
666    };
667    let builder = RealmBuilder::with_params(realm_param).await?;
668    let wrapper_realm =
669        builder.add_child_realm(WRAPPER_REALM_NAME, ChildOptions::new().eager()).await?;
670
671    let hermetic_test_package_name = Arc::new(test_package.to_owned());
672    let other_allowed_packages = get_allowed_package_value(&suite_facet);
673
674    let hermetic_test_package_name_clone = hermetic_test_package_name.clone();
675    let other_allowed_packages_clone = other_allowed_packages.clone();
676    let resolver = wrapper_realm
677        .add_local_child(
678            HERMETIC_RESOLVER_REALM_NAME,
679            move |handles| {
680                Box::pin(resolver::serve_hermetic_resolver(
681                    handles,
682                    hermetic_test_package_name_clone.clone(),
683                    other_allowed_packages_clone.clone(),
684                    resolver.clone(),
685                    pkg_resolver.clone(),
686                ))
687            },
688            ChildOptions::new(),
689        )
690        .await?;
691
692    // Provide and expose the debug data capability to the test environment.
693    let owned_url = test_url.to_string();
694    let mock_ready_event = async_utils::event::Event::new();
695    let mock_ready_clone = mock_ready_event.clone();
696    let debug_data = wrapper_realm
697        .add_local_child(
698            DEBUG_DATA_REALM_NAME,
699            move |handles| {
700                Box::pin(serve_debug_data_publisher(
701                    handles,
702                    owned_url.clone(),
703                    debug_data_sender.clone(),
704                    mock_ready_clone.clone(),
705                ))
706            },
707            // This component is launched eagerly so that debug_data always signals ready, even
708            // in absence of a connection request. TODO(https://fxbug.dev/42056523): Remove once
709            // synchronization is provided by cm.
710            ChildOptions::new().eager(),
711        )
712        .await?;
713    let mut debug_data_decl = wrapper_realm.get_component_decl(&debug_data).await?;
714    debug_data_decl.exposes.push(cm_rust::ExposeDecl::Protocol(cm_rust::ExposeProtocolDecl {
715        source: cm_rust::ExposeSource::Self_,
716        source_name: "fuchsia.debugdata.Publisher".parse().unwrap(),
717        source_dictionary: Default::default(),
718        target: cm_rust::ExposeTarget::Parent,
719        target_name: "fuchsia.debugdata.Publisher".parse().unwrap(),
720        availability: cm_rust::Availability::Required,
721    }));
722    debug_data_decl.capabilities.push(cm_rust::CapabilityDecl::Protocol(cm_rust::ProtocolDecl {
723        name: "fuchsia.debugdata.Publisher".parse().unwrap(),
724        source_path: Some("/svc/fuchsia.debugdata.Publisher".parse().unwrap()),
725        delivery: Default::default(),
726    }));
727    wrapper_realm.replace_component_decl(&debug_data, debug_data_decl).await?;
728
729    // Provide and expose the resolver capability from the resolver to test_wrapper.
730    let mut hermetic_resolver_decl =
731        wrapper_realm.get_component_decl(HERMETIC_RESOLVER_REALM_NAME).await?;
732    hermetic_resolver_decl.exposes.push(cm_rust::ExposeDecl::Resolver(
733        cm_rust::ExposeResolverDecl {
734            source: cm_rust::ExposeSource::Self_,
735            source_name: HERMETIC_RESOLVER_CAPABILITY_NAME.parse().unwrap(),
736            source_dictionary: Default::default(),
737            target: cm_rust::ExposeTarget::Parent,
738            target_name: HERMETIC_RESOLVER_CAPABILITY_NAME.parse().unwrap(),
739        },
740    ));
741    hermetic_resolver_decl.capabilities.push(cm_rust::CapabilityDecl::Resolver(
742        cm_rust::ResolverDecl {
743            name: HERMETIC_RESOLVER_CAPABILITY_NAME.parse().unwrap(),
744            source_path: Some("/svc/fuchsia.component.resolution.Resolver".parse().unwrap()),
745        },
746    ));
747
748    wrapper_realm
749        .replace_component_decl(HERMETIC_RESOLVER_REALM_NAME, hermetic_resolver_decl)
750        .await?;
751
752    let memfs = wrapper_realm
753        .add_child(MEMFS_REALM_NAME, MEMFS_FOR_EMBEDDING_URL, ChildOptions::new())
754        .await?;
755
756    // Create the hermetic environment in the test_wrapper.
757    let mut test_wrapper_decl = wrapper_realm.get_realm_decl().await?;
758    test_wrapper_decl.environments.push(cm_rust::EnvironmentDecl {
759        name: TEST_ENVIRONMENT_NAME.parse().unwrap(),
760        extends: fdecl::EnvironmentExtends::Realm,
761        resolvers: vec![cm_rust::ResolverRegistration {
762            resolver: HERMETIC_RESOLVER_CAPABILITY_NAME.parse().unwrap(),
763            source: cm_rust::RegistrationSource::Child(String::from(
764                HERMETIC_RESOLVER_CAPABILITY_NAME,
765            )),
766            scheme: String::from("fuchsia-pkg"),
767        }],
768        runners: vec![],
769        debug_capabilities: vec![cm_rust::DebugRegistration::Protocol(
770            cm_rust::DebugProtocolRegistration {
771                source_name: "fuchsia.debugdata.Publisher".parse().unwrap(),
772                source: cm_rust::RegistrationSource::Child(DEBUG_DATA_REALM_NAME.to_string()),
773                target_name: "fuchsia.debugdata.Publisher".parse().unwrap(),
774            },
775        )],
776        stop_timeout_ms: None,
777    });
778
779    // Add the collection to hold the test. This lets us individually destroy the test.
780    test_wrapper_decl.collections.push(cm_rust::CollectionDecl {
781        name: TEST_ROOT_COLLECTION.parse().unwrap(),
782        durability: fdecl::Durability::Transient,
783        environment: Some(TEST_ENVIRONMENT_NAME.parse().unwrap()),
784        allowed_offers: cm_types::AllowedOffers::StaticOnly,
785        allow_long_names: false,
786        persistent_storage: None,
787    });
788
789    test_wrapper_decl.capabilities.push(cm_rust::CapabilityDecl::Storage(cm_rust::StorageDecl {
790        name: CUSTOM_ARTIFACTS_CAPABILITY_NAME.parse().unwrap(),
791        source: cm_rust::StorageDirectorySource::Child(MEMFS_REALM_NAME.to_string()),
792        backing_dir: "memfs".parse().unwrap(),
793        subdir: "custom_artifacts".parse().unwrap(),
794        storage_id: fdecl::StorageId::StaticInstanceIdOrMoniker,
795    }));
796
797    test_wrapper_decl.capabilities.push(cm_rust::CapabilityDecl::Dictionary(
798        cm_rust::DictionaryDecl {
799            name: DIAGNOSTICS_DICTIONARY_NAME.parse().unwrap(),
800            source_path: None,
801        },
802    ));
803
804    wrapper_realm.replace_realm_decl(test_wrapper_decl).await?;
805
806    let test_root = Ref::collection(TEST_ROOT_COLLECTION);
807    let archivist = wrapper_realm
808        .add_child(ARCHIVIST_REALM_NAME, ARCHIVIST_FOR_EMBEDDING_URL, ChildOptions::new().eager())
809        .await?;
810
811    // Parent to archivist
812    builder
813        .add_route(
814            Route::new()
815                .capability(Capability::dictionary(DIAGNOSTICS_DICTIONARY_NAME))
816                .from(Ref::parent())
817                .to(&wrapper_realm),
818        )
819        .await?;
820
821    wrapper_realm
822        .add_route(
823            Route::new()
824                .capability(Capability::dictionary(DIAGNOSTICS_DICTIONARY_NAME))
825                .from(Ref::parent())
826                .to(&archivist)
827                .to(&memfs),
828        )
829        .await?;
830
831    // archivist to test root
832    wrapper_realm
833        .add_route(
834            Route::new()
835                .capability(Capability::protocol_by_name("fuchsia.logger.Log"))
836                .from(&archivist)
837                .to(test_root.clone()),
838        )
839        .await?;
840    wrapper_realm
841        .add_route(
842            Route::new()
843                .capability(Capability::protocol_by_name("fuchsia.logger.LogSink"))
844                .capability(Capability::protocol_by_name("fuchsia.inspect.InspectSink"))
845                .from(&archivist)
846                .to(test_root.clone())
847                .to(Ref::dictionary(format!("self/{DIAGNOSTICS_DICTIONARY_NAME}")))
848                .to(&resolver),
849        )
850        .await?;
851    wrapper_realm
852        .add_route(
853            Route::new()
854                .capability(Capability::protocol_by_name("fuchsia.debugdata.Publisher"))
855                .from(&debug_data)
856                .to(Ref::dictionary(format!("self/{DIAGNOSTICS_DICTIONARY_NAME}"))),
857        )
858        .await?;
859
860    // Diagnostics dictionary to resolver and test_root
861    wrapper_realm
862        .add_route(
863            Route::new()
864                .capability(Capability::dictionary(DIAGNOSTICS_DICTIONARY_NAME))
865                .from(Ref::self_())
866                .to(test_root.clone())
867                .to(&resolver),
868        )
869        .await?;
870
871    // archivist to parent and test root
872    wrapper_realm
873        .add_route(
874            Route::new()
875                .capability(Capability::protocol::<fdiagnostics::ArchiveAccessorMarker>())
876                .capability(Capability::protocol::<
877                    fidl_fuchsia_diagnostics_host::ArchiveAccessorMarker,
878                >())
879                .from_dictionary("diagnostics-accessors")
880                .from(&archivist)
881                .to(Ref::parent())
882                .to(test_root.clone()),
883        )
884        .await?;
885
886    wrapper_realm
887        .add_route(
888            Route::new()
889                .capability(Capability::protocol::<fdiagnostics::LogSettingsMarker>())
890                .from(&archivist)
891                .to(Ref::parent()),
892        )
893        .await?;
894
895    wrapper_realm
896        .add_route(
897            Route::new()
898                .capability(Capability::storage(CUSTOM_ARTIFACTS_CAPABILITY_NAME))
899                .from(Ref::self_())
900                .to(test_root.clone()),
901        )
902        .await?;
903
904    wrapper_realm
905        .add_route(
906            Route::new()
907                .capability(Capability::protocol::<fsys::StorageAdminMarker>())
908                .from(Ref::capability(CUSTOM_ARTIFACTS_CAPABILITY_NAME))
909                .to(Ref::parent()),
910        )
911        .await?;
912
913    // We need to expose the raw protocols so that the driver test realm's driver framework
914    // can use it in order to provide support for subpackaged drivers.
915    wrapper_realm
916        .add_route(
917            Route::new()
918                .capability(
919                    Capability::protocol_by_name("fuchsia.component.resolution.Resolver-hermetic")
920                        .path("/svc/fuchsia.component.resolution.Resolver"),
921                )
922                .from(&resolver)
923                .to(test_root.clone()),
924        )
925        .await?;
926    wrapper_realm
927        .add_route(
928            Route::new()
929                .capability(
930                    Capability::protocol_by_name("fuchsia.pkg.PackageResolver-hermetic")
931                        .path("/svc/fuchsia.pkg.PackageResolver"),
932                )
933                .from(&resolver)
934                .to(test_root.clone()),
935        )
936        .await?;
937
938    // wrapper realm to parent
939    wrapper_realm
940        .add_route(
941            Route::new()
942                .capability(Capability::protocol::<fcomponent::RealmMarker>())
943                .from(Ref::framework())
944                .to(Ref::parent()),
945        )
946        .await?;
947
948    // wrapper realm to archivist
949
950    wrapper_realm
951        .add_route(
952            Route::new()
953                .capability(
954                    Capability::event_stream("capability_requested").with_scope(test_root.clone()),
955                )
956                .from(Ref::parent())
957                .to(&archivist),
958        )
959        .await?;
960
961    builder
962        .add_route(
963            Route::new()
964                // from archivist
965                .capability(Capability::protocol::<fdiagnostics::ArchiveAccessorMarker>())
966                .capability(Capability::protocol::<
967                    fidl_fuchsia_diagnostics_host::ArchiveAccessorMarker,
968                >())
969                .capability(Capability::protocol::<fdiagnostics::LogSettingsMarker>())
970                // from test root
971                .capability(Capability::protocol::<ftest::SuiteMarker>())
972                // custom_artifact capability
973                .capability(Capability::protocol::<fsys::StorageAdminMarker>())
974                .capability(Capability::protocol::<fcomponent::RealmMarker>())
975                .from(&wrapper_realm)
976                .to(Ref::parent()),
977        )
978        .await?;
979    if let Some(suite_realm) = suite_realm {
980        apply_offers(&builder, &wrapper_realm, &suite_realm.offers).await?;
981    } else {
982        above_root_capabilities_for_test
983            .apply(suite_facet.collection, &builder, &wrapper_realm)
984            .await?;
985    }
986
987    Ok((builder, mock_ready_event))
988}
989
990fn get_invocation_options(options: ftest_manager::RunOptions) -> ftest::RunOptions {
991    ftest::RunOptions {
992        include_disabled_tests: options.run_disabled_tests,
993        parallel: options.parallel,
994        arguments: options.arguments,
995        break_on_failure: options.break_on_failure,
996        ..Default::default()
997    }
998}
999
1000/// Runs the test component using `suite` and collects stdout logs and results.
1001async fn run_invocations(
1002    suite: &ftest::SuiteProxy,
1003    invocations: Vec<Invocation>,
1004    run_options: fidl_fuchsia_test::RunOptions,
1005    counter: &AtomicU32,
1006    sender: &mut mpsc::Sender<Result<SuiteEvents, LaunchError>>,
1007    timeout_fut: futures::future::Shared<fasync::Timer>,
1008) -> Result<SuiteStatus, anyhow::Error> {
1009    let (run_listener_client, mut run_listener) = fidl::endpoints::create_request_stream();
1010    suite.run(&invocations, &run_options, run_listener_client)?;
1011
1012    let tasks = Arc::new(lock::Mutex::new(vec![]));
1013    let running_test_cases = Arc::new(lock::Mutex::new(HashSet::new()));
1014    let tasks_clone = tasks.clone();
1015    let initial_suite_status: SuiteStatus;
1016    let mut sender_clone = sender.clone();
1017    let test_fut = async {
1018        let mut initial_suite_status = SuiteStatus::DidNotFinish;
1019        while let Some(result_event) =
1020            run_listener.try_next().await.context("error waiting for listener")?
1021        {
1022            match result_event {
1023                ftest::RunListenerRequest::OnTestCaseStarted {
1024                    invocation,
1025                    std_handles,
1026                    listener,
1027                    control_handle: _,
1028                } => {
1029                    let name =
1030                        invocation.name.ok_or(format_err!("cannot find name in invocation"))?;
1031                    let identifier = counter.fetch_add(1, Ordering::Relaxed);
1032                    let events = vec![
1033                        Ok(SuiteEvents::case_found(identifier, name).into()),
1034                        Ok(SuiteEvents::case_started(identifier).into()),
1035                    ];
1036                    for event in events {
1037                        sender_clone.send(event).await.unwrap();
1038                    }
1039                    let listener = listener.into_stream();
1040                    running_test_cases.lock().await.insert(identifier);
1041                    let running_test_cases = running_test_cases.clone();
1042                    let mut sender = sender_clone.clone();
1043                    let task = fasync::Task::spawn(async move {
1044                        let _ = &std_handles;
1045                        let mut sender_stdout = sender.clone();
1046                        let mut sender_stderr = sender.clone();
1047                        let completion_fut = async {
1048                            let status = listen_for_completion(listener).await;
1049                            sender
1050                                .send(Ok(
1051                                    SuiteEvents::case_stopped(identifier, status.clone()).into()
1052                                ))
1053                                .await
1054                                .unwrap();
1055                            status
1056                        };
1057                        let ftest::StdHandles { out, err, .. } = std_handles;
1058                        let stdout_fut = async move {
1059                            if let Some(out) = out {
1060                                match fasync::OnSignals::new(
1061                                    &out,
1062                                    zx::Signals::SOCKET_READABLE | zx::Signals::SOCKET_PEER_CLOSED,
1063                                )
1064                                .await
1065                                {
1066                                    Ok(signals)
1067                                        if signals.contains(zx::Signals::SOCKET_READABLE) =>
1068                                    {
1069                                        sender_stdout
1070                                            .send(Ok(
1071                                                SuiteEvents::case_stdout(identifier, out).into()
1072                                            ))
1073                                            .await
1074                                            .unwrap();
1075                                    }
1076                                    Ok(_) | Err(_) => (),
1077                                }
1078                            }
1079                        };
1080                        let stderr_fut = async move {
1081                            if let Some(err) = err {
1082                                match fasync::OnSignals::new(
1083                                    &err,
1084                                    zx::Signals::SOCKET_READABLE | zx::Signals::SOCKET_PEER_CLOSED,
1085                                )
1086                                .await
1087                                {
1088                                    Ok(signals)
1089                                        if signals.contains(zx::Signals::SOCKET_READABLE) =>
1090                                    {
1091                                        sender_stderr
1092                                            .send(Ok(
1093                                                SuiteEvents::case_stderr(identifier, err).into()
1094                                            ))
1095                                            .await
1096                                            .unwrap();
1097                                    }
1098                                    Ok(_) | Err(_) => (),
1099                                }
1100                            }
1101                        };
1102                        let (status, (), ()) =
1103                            futures::future::join3(completion_fut, stdout_fut, stderr_fut).await;
1104
1105                        sender
1106                            .send(Ok(SuiteEvents::case_finished(identifier).into()))
1107                            .await
1108                            .unwrap();
1109                        running_test_cases.lock().await.remove(&identifier);
1110                        status
1111                    });
1112                    tasks_clone.lock().await.push(task);
1113                }
1114                ftest::RunListenerRequest::OnFinished { .. } => {
1115                    initial_suite_status = SuiteStatus::Passed;
1116                    break;
1117                }
1118            }
1119        }
1120        Ok(initial_suite_status)
1121    }
1122    .fuse();
1123
1124    futures::pin_mut!(test_fut);
1125    let timeout_fut = timeout_fut.fuse();
1126    futures::pin_mut!(timeout_fut);
1127
1128    futures::select! {
1129        () = timeout_fut => {
1130            let mut all_tasks = vec![];
1131            let mut tasks = tasks.lock().await;
1132            all_tasks.append(&mut tasks);
1133            drop(tasks);
1134            drop(all_tasks);
1135            let running_test_cases = running_test_cases.lock().await;
1136            for i in &*running_test_cases {
1137                sender
1138                    .send(Ok(SuiteEvents::case_stopped(*i, CaseStatus::TimedOut).into()))
1139                    .await
1140                    .unwrap();
1141                sender
1142                    .send(Ok(SuiteEvents::case_finished(*i).into()))
1143                    .await
1144                    .unwrap();
1145            }
1146            return Ok(SuiteStatus::TimedOut);
1147        }
1148        r = test_fut => {
1149            initial_suite_status = match r {
1150                Err(e) => {
1151                    return Err(e);
1152                }
1153                Ok(s) => s,
1154            };
1155        }
1156    }
1157
1158    let mut tasks = tasks.lock().await;
1159    let mut all_tasks = vec![];
1160    all_tasks.append(&mut tasks);
1161    // await for all invocations to complete for which test case never completed.
1162    let suite_status = join_all(all_tasks)
1163        .await
1164        .into_iter()
1165        .fold(initial_suite_status, get_suite_status_from_case_status);
1166    Ok(suite_status)
1167}
1168
1169fn concat_suite_status(initial: SuiteStatus, new: SuiteStatus) -> SuiteStatus {
1170    if initial.into_primitive() > new.into_primitive() {
1171        return initial;
1172    }
1173    return new;
1174}
1175
1176fn get_suite_status_from_case_status(
1177    initial_suite_status: SuiteStatus,
1178    case_status: CaseStatus,
1179) -> SuiteStatus {
1180    let status = match case_status {
1181        CaseStatus::Passed => SuiteStatus::Passed,
1182        CaseStatus::Failed => SuiteStatus::Failed,
1183        CaseStatus::TimedOut => SuiteStatus::TimedOut,
1184        CaseStatus::Skipped => SuiteStatus::Passed,
1185        CaseStatus::Error => SuiteStatus::DidNotFinish,
1186        _ => {
1187            panic!("this should not happen");
1188        }
1189    };
1190    concat_suite_status(initial_suite_status, status)
1191}
1192
1193fn enumeration_error(err: fidl::Error) -> anyhow::Error {
1194    match err {
1195        fidl::Error::ClientChannelClosed { .. } => anyhow::anyhow!(
1196            "The test protocol was closed during enumeration. This may mean that the test is using
1197            wrong test runner or `fuchsia.test.Suite` was not configured correctly. Refer to: \
1198            https://fuchsia.dev/go/components/test-errors"
1199        ),
1200        err => err.into(),
1201    }
1202}
1203
1204/// Listen for test completion on the given |listener|. Returns None if the channel is closed
1205/// before a test completion event.
1206async fn listen_for_completion(mut listener: ftest::CaseListenerRequestStream) -> CaseStatus {
1207    match listener.try_next().await.context("waiting for listener") {
1208        Ok(Some(request)) => {
1209            let ftest::CaseListenerRequest::Finished { result, control_handle: _ } = request;
1210            let result = match result.status {
1211                Some(status) => match status {
1212                    fidl_fuchsia_test::Status::Passed => CaseStatus::Passed,
1213                    fidl_fuchsia_test::Status::Failed => CaseStatus::Failed,
1214                    fidl_fuchsia_test::Status::Skipped => CaseStatus::Skipped,
1215                },
1216                // This will happen when test protocol is not properly implemented
1217                // by the test and it forgets to set the result.
1218                None => CaseStatus::Error,
1219            };
1220            result
1221        }
1222        Err(e) => {
1223            warn!("listener failed: {:?}", e);
1224            CaseStatus::Error
1225        }
1226        Ok(None) => CaseStatus::Error,
1227    }
1228}
1229
1230#[cfg(test)]
1231mod tests {
1232    use crate::constants::{HERMETIC_TESTS_COLLECTION, SYSTEM_TESTS_COLLECTION};
1233
1234    use super::*;
1235    use maplit::hashset;
1236
1237    #[test]
1238    fn case_matcher_tests() {
1239        let all_test_case_names = hashset! {
1240            "Foo.Test1", "Foo.Test2", "Foo.Test3", "Bar.Test1", "Bar.Test2", "Bar.Test3"
1241        };
1242
1243        let cases = vec![
1244            (vec![], all_test_case_names.clone()),
1245            (vec!["Foo.Test1"], hashset! {"Foo.Test1"}),
1246            (vec!["Foo.*"], hashset! {"Foo.Test1", "Foo.Test2", "Foo.Test3"}),
1247            (vec!["-Foo.*"], hashset! {"Bar.Test1", "Bar.Test2", "Bar.Test3"}),
1248            (vec!["Foo.*", "-*.Test2"], hashset! {"Foo.Test1", "Foo.Test3"}),
1249        ];
1250
1251        for (filters, expected_matching_cases) in cases.into_iter() {
1252            let case_matcher = CaseMatcher::new(&filters).expect("Create case matcher");
1253            for test_case in all_test_case_names.iter() {
1254                match expected_matching_cases.contains(test_case) {
1255                    true => assert!(
1256                        case_matcher.matches(test_case),
1257                        "Expected filters {:?} to match test case name {}",
1258                        filters,
1259                        test_case
1260                    ),
1261                    false => assert!(
1262                        !case_matcher.matches(test_case),
1263                        "Expected filters {:?} to not match test case name {}",
1264                        filters,
1265                        test_case
1266                    ),
1267                }
1268            }
1269        }
1270    }
1271
1272    #[test]
1273    fn suite_status() {
1274        let all_case_status = vec![
1275            CaseStatus::Error,
1276            CaseStatus::TimedOut,
1277            CaseStatus::Failed,
1278            CaseStatus::Skipped,
1279            CaseStatus::Passed,
1280        ];
1281        for status in &all_case_status {
1282            assert_eq!(
1283                get_suite_status_from_case_status(SuiteStatus::InternalError, *status),
1284                SuiteStatus::InternalError
1285            );
1286        }
1287
1288        for status in &all_case_status {
1289            let s = get_suite_status_from_case_status(SuiteStatus::TimedOut, *status);
1290            assert_eq!(s, SuiteStatus::TimedOut);
1291        }
1292
1293        for status in &all_case_status {
1294            let s = get_suite_status_from_case_status(SuiteStatus::DidNotFinish, *status);
1295            let mut expected = SuiteStatus::DidNotFinish;
1296            if status == &CaseStatus::TimedOut {
1297                expected = SuiteStatus::TimedOut;
1298            }
1299            assert_eq!(s, expected);
1300        }
1301
1302        for status in &all_case_status {
1303            let s = get_suite_status_from_case_status(SuiteStatus::Failed, *status);
1304            let expected = match *status {
1305                CaseStatus::TimedOut => SuiteStatus::TimedOut,
1306                CaseStatus::Error => SuiteStatus::DidNotFinish,
1307                _ => SuiteStatus::Failed,
1308            };
1309            assert_eq!(s, expected);
1310        }
1311
1312        for status in &all_case_status {
1313            let s = get_suite_status_from_case_status(SuiteStatus::Passed, *status);
1314            let mut expected = SuiteStatus::Passed;
1315            if status == &CaseStatus::Error {
1316                expected = SuiteStatus::DidNotFinish;
1317            }
1318            if status == &CaseStatus::TimedOut {
1319                expected = SuiteStatus::TimedOut;
1320            }
1321            if status == &CaseStatus::Failed {
1322                expected = SuiteStatus::Failed;
1323            }
1324            assert_eq!(s, expected);
1325        }
1326
1327        let all_suite_status = vec![
1328            SuiteStatus::Passed,
1329            SuiteStatus::Failed,
1330            SuiteStatus::TimedOut,
1331            SuiteStatus::Stopped,
1332            SuiteStatus::InternalError,
1333        ];
1334
1335        for initial_status in &all_suite_status {
1336            for status in &all_suite_status {
1337                let s = concat_suite_status(*initial_status, *status);
1338                let expected: SuiteStatus;
1339                if initial_status.into_primitive() > status.into_primitive() {
1340                    expected = *initial_status;
1341                } else {
1342                    expected = *status;
1343                }
1344
1345                assert_eq!(s, expected);
1346            }
1347        }
1348    }
1349
1350    #[test]
1351    fn test_allowed_packages_value() {
1352        let mut suite_facet = facet::SuiteFacets {
1353            collection: HERMETIC_TESTS_COLLECTION,
1354            deprecated_allowed_packages: None,
1355        };
1356
1357        // default for hermetic realm is no other allowed package.
1358        assert_eq!(AllowedPackages::zero_allowed_pkgs(), get_allowed_package_value(&suite_facet));
1359
1360        // deprecated_allowed_packages overrides HERMETIC_TESTS_COLLECTION
1361        suite_facet.deprecated_allowed_packages = Some(vec!["pkg-one".to_owned()]);
1362        assert_eq!(
1363            AllowedPackages::from_iter(["pkg-one".to_owned()]),
1364            get_allowed_package_value(&suite_facet)
1365        );
1366
1367        // test with other collections
1368        suite_facet.collection = SYSTEM_TESTS_COLLECTION;
1369        assert_eq!(
1370            AllowedPackages::from_iter(["pkg-one".to_owned()]),
1371            get_allowed_package_value(&suite_facet)
1372        );
1373
1374        // test default with other collection
1375        suite_facet.deprecated_allowed_packages = None;
1376        assert_eq!(AllowedPackages::zero_allowed_pkgs(), get_allowed_package_value(&suite_facet));
1377    }
1378}