test_runners_test_lib/
test_lib.rs

1// Copyright 2020 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use anyhow::{format_err, Context, Error};
6use fidl::endpoints;
7use fidl::endpoints::{ClientEnd, Proxy};
8use fidl_fuchsia_test::CaseListenerRequest::Finished;
9use fidl_fuchsia_test::RunListenerRequest::{OnFinished, OnTestCaseStarted};
10use fidl_fuchsia_test::{Invocation, Result_ as TestResult, RunListenerRequestStream};
11use fuchsia_component::client::{self, connect_to_protocol_at_dir_root};
12use fuchsia_runtime::job_default;
13use futures::channel::mpsc;
14use futures::prelude::*;
15use namespace::{Namespace, NamespaceError};
16use std::collections::HashMap;
17use std::sync::Arc;
18use test_manager_test_lib::RunEvent;
19use test_runners_lib::elf::{BuilderArgs, Component};
20use {
21    fidl_fuchsia_component as fcomponent, fidl_fuchsia_component_decl as fdecl,
22    fidl_fuchsia_component_runner as fcrunner, fidl_fuchsia_io as fio,
23    fidl_fuchsia_test_manager as ftest_manager, fuchsia_async as fasync,
24};
25
26#[derive(PartialEq, Debug)]
27pub enum ListenerEvent {
28    StartTest(String),
29    FinishTest(String, TestResult),
30    FinishAllTests,
31}
32
33fn get_ord_index_and_name(event: &ListenerEvent) -> (usize, &str) {
34    match event {
35        ListenerEvent::StartTest(name) => (0, name),
36        ListenerEvent::FinishTest(name, _) => (1, name),
37        ListenerEvent::FinishAllTests => (2, ""),
38    }
39}
40
41// Orders by test name and then event type.
42impl Ord for ListenerEvent {
43    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
44        let (s_index, s_test_name) = get_ord_index_and_name(self);
45        let (o_index, o_test_name) = get_ord_index_and_name(other);
46        if s_test_name == o_test_name || s_index == 2 || o_index == 2 {
47            return s_index.cmp(&o_index);
48        }
49        return s_test_name.cmp(&o_test_name);
50    }
51}
52
53// Makes sure that FinishTest event never shows up before StartTest and FinishAllTests is always
54// last.
55pub fn assert_event_ord(events: &Vec<ListenerEvent>) {
56    let mut tests = HashMap::new();
57    let mut all_finish = false;
58    for event in events {
59        assert!(!all_finish, "got FinishAllTests event twice: {:#?}", events);
60        match event {
61            ListenerEvent::StartTest(name) => {
62                assert!(
63                    !tests.contains_key(&name),
64                    "Multiple StartTest for test {}: {:#?}",
65                    name,
66                    events
67                );
68                tests.insert(name, false);
69            }
70            ListenerEvent::FinishTest(name, _) => {
71                assert!(
72                    tests.contains_key(&name),
73                    "Got finish before start event for test {}: {:#?}",
74                    name,
75                    events
76                );
77                assert!(
78                    !tests.insert(name, true).unwrap(),
79                    "Multiple FinishTest for test {}: {:#?}",
80                    name,
81                    events
82                );
83            }
84            ListenerEvent::FinishAllTests => {
85                all_finish = true;
86            }
87        }
88    }
89}
90
91impl PartialOrd for ListenerEvent {
92    fn partial_cmp(&self, other: &ListenerEvent) -> Option<core::cmp::Ordering> {
93        Some(self.cmp(other))
94    }
95}
96
97impl Eq for ListenerEvent {}
98
99impl ListenerEvent {
100    pub fn start_test(name: &str) -> ListenerEvent {
101        ListenerEvent::StartTest(name.to_string())
102    }
103    pub fn finish_test(name: &str, test_result: TestResult) -> ListenerEvent {
104        ListenerEvent::FinishTest(name.to_string(), test_result)
105    }
106    pub fn finish_all_test() -> ListenerEvent {
107        ListenerEvent::FinishAllTests
108    }
109}
110
111impl Clone for ListenerEvent {
112    fn clone(&self) -> Self {
113        match self {
114            ListenerEvent::StartTest(name) => ListenerEvent::start_test(name),
115            ListenerEvent::FinishTest(name, test_result) => ListenerEvent::finish_test(
116                name,
117                TestResult { status: test_result.status.clone(), ..Default::default() },
118            ),
119            ListenerEvent::FinishAllTests => ListenerEvent::finish_all_test(),
120        }
121    }
122}
123
124/// Collects all the listener event as they come and return in a vector.
125pub async fn collect_listener_event(
126    mut listener: RunListenerRequestStream,
127) -> Result<Vec<ListenerEvent>, Error> {
128    let mut ret = vec![];
129    // collect loggers so that they do not die.
130    let mut loggers = vec![];
131    while let Some(result_event) = listener.try_next().await? {
132        match result_event {
133            OnTestCaseStarted { invocation, std_handles, listener, .. } => {
134                let name = invocation.name.unwrap();
135                ret.push(ListenerEvent::StartTest(name.clone()));
136                loggers.push(std_handles);
137                let mut listener = listener.into_stream();
138                // We want exhaustive match, and if we add more variants in the future we'd need to
139                // handle the requests in a loop, so allow this lint violation.
140                #[allow(clippy::never_loop)]
141                while let Some(result) = listener.try_next().await? {
142                    match result {
143                        Finished { result, .. } => {
144                            ret.push(ListenerEvent::FinishTest(name, result));
145                            break;
146                        }
147                    }
148                }
149            }
150            OnFinished { .. } => {
151                ret.push(ListenerEvent::FinishAllTests);
152                break;
153            }
154        }
155    }
156    Ok(ret)
157}
158
159/// Helper method to convert names to `Invocation`.
160pub fn names_to_invocation(names: Vec<&str>) -> Vec<Invocation> {
161    names
162        .iter()
163        .map(|s| Invocation { name: Some(s.to_string()), tag: None, ..Default::default() })
164        .collect()
165}
166
167// process events by parsing and normalizing logs. Returns `RunEvents` and collected logs.
168pub async fn process_events(
169    suite_instance: test_manager_test_lib::SuiteRunInstance,
170    exclude_empty_logs: bool,
171) -> Result<(Vec<RunEvent>, Vec<String>), Error> {
172    let (sender, mut recv) = mpsc::channel(1);
173    let execution_task = fasync::Task::spawn(async move {
174        suite_instance.collect_events_with_watch(sender, true, true).await
175    });
176    let mut events = vec![];
177    let mut log_tasks = vec![];
178    let mut buffered_stdout = HashMap::new();
179    let mut buffered_stderr = HashMap::new();
180    while let Some(event) = recv.next().await {
181        match event.payload {
182            test_manager_test_lib::SuiteEventPayload::RunEvent(RunEvent::CaseStdout {
183                name,
184                stdout_message,
185            }) => {
186                let strings = line_buffer_std_message(
187                    &name,
188                    stdout_message,
189                    exclude_empty_logs,
190                    &mut buffered_stdout,
191                );
192                for s in strings {
193                    events.push(RunEvent::case_stdout(name.clone(), s));
194                }
195            }
196            test_manager_test_lib::SuiteEventPayload::RunEvent(RunEvent::CaseStderr {
197                name,
198                stderr_message,
199            }) => {
200                let strings = line_buffer_std_message(
201                    &name,
202                    stderr_message,
203                    exclude_empty_logs,
204                    &mut buffered_stderr,
205                );
206                for s in strings {
207                    events.push(RunEvent::case_stderr(name.clone(), s));
208                }
209            }
210            test_manager_test_lib::SuiteEventPayload::RunEvent(e) => events.push(e),
211            test_manager_test_lib::SuiteEventPayload::SuiteLog { log_stream } => {
212                let t = fasync::Task::spawn(log_stream.collect::<Vec<_>>());
213                log_tasks.push(t);
214            }
215            test_manager_test_lib::SuiteEventPayload::TestCaseLog { .. } => {
216                panic!("not supported yet!")
217            }
218            test_manager_test_lib::SuiteEventPayload::DebugData { .. } => {
219                panic!("not supported yet!")
220            }
221        }
222    }
223    execution_task.await.context("test execution failed")?;
224
225    for (name, log) in buffered_stdout {
226        events.push(RunEvent::case_stdout(name, log));
227    }
228    for (name, log) in buffered_stderr {
229        events.push(RunEvent::case_stderr(name, log));
230    }
231
232    let mut collected_logs = vec![];
233    for t in log_tasks {
234        let logs = t.await;
235        for log_result in logs {
236            let log = log_result?;
237            collected_logs.push(log.msg().unwrap().to_string());
238        }
239    }
240
241    Ok((events, collected_logs))
242}
243
244// Process stdout/stderr messages and return Vec of processed strings
245fn line_buffer_std_message(
246    name: &str,
247    std_message: String,
248    exclude_empty_logs: bool,
249    buffer: &mut HashMap<String, String>,
250) -> Vec<String> {
251    let mut ret = vec![];
252    let logs = std_message.split("\n");
253    let mut logs = logs.collect::<Vec<&str>>();
254    // discard last empty log(if it ended in newline, or  store im-complete line)
255    let mut last_incomplete_line = logs.pop();
256    if std_message.as_bytes().last() == Some(&b'\n') {
257        last_incomplete_line = None;
258    }
259    for log in logs {
260        if exclude_empty_logs && log.len() == 0 {
261            continue;
262        }
263        let mut msg = log.to_owned();
264        // This is only executed for first log line and used to concat previous
265        // buffered line.
266        if let Some(prev_log) = buffer.remove(name) {
267            msg = format!("{}{}", prev_log, msg);
268        }
269        ret.push(msg);
270    }
271    if let Some(log) = last_incomplete_line {
272        let mut log = log.to_owned();
273        if let Some(prev_log) = buffer.remove(name) {
274            log = format!("{}{}", prev_log, log);
275        }
276        buffer.insert(name.to_string(), log);
277    }
278    ret
279}
280
281// Binds to test manager component and returns suite runner service.
282pub async fn connect_to_suite_runner() -> Result<ftest_manager::SuiteRunnerProxy, Error> {
283    let realm = client::connect_to_protocol::<fcomponent::RealmMarker>()
284        .context("could not connect to Realm service")?;
285
286    let child_ref = fdecl::ChildRef { name: "test_manager".to_owned(), collection: None };
287    let (dir, server_end) = endpoints::create_proxy::<fio::DirectoryMarker>();
288    realm
289        .open_exposed_dir(&child_ref, server_end)
290        .await
291        .context("open_exposed_dir fidl call failed for test manager")?
292        .map_err(|e| format_err!("failed to create test manager: {:?}", e))?;
293
294    connect_to_protocol_at_dir_root::<ftest_manager::SuiteRunnerMarker>(&dir)
295        .context("failed to open test suite runner service")
296}
297
298fn create_ns_from_current_ns(
299    dir_paths: Vec<(&str, fio::Flags)>,
300) -> Result<Namespace, NamespaceError> {
301    let mut ns = vec![];
302    for (path, permission) in dir_paths {
303        let chan = fuchsia_fs::directory::open_in_namespace(path, permission)
304            .unwrap()
305            .into_channel()
306            .unwrap()
307            .into_zx_channel();
308        let handle = ClientEnd::new(chan);
309
310        ns.push(fcrunner::ComponentNamespaceEntry {
311            path: Some(path.to_string()),
312            directory: Some(handle),
313            ..Default::default()
314        });
315    }
316    Namespace::try_from(ns)
317}
318
319/// Create a new component object for testing purposes.
320pub async fn test_component(
321    url: &str,
322    name: &str,
323    binary: &str,
324    args: Vec<String>,
325) -> Result<Arc<Component>, Error> {
326    let ns = create_ns_from_current_ns(vec![
327        ("/pkg", fuchsia_fs::PERM_READABLE | fuchsia_fs::PERM_EXECUTABLE),
328        // TODO(b/376735013): Restrict this to LogSink instead of all of /svc.
329        ("/svc", fuchsia_fs::PERM_READABLE | fuchsia_fs::PERM_EXECUTABLE),
330    ])?;
331    let component = Component::create_for_tests(BuilderArgs {
332        url: url.to_string(),
333        name: name.to_string(),
334        binary: binary.to_string(),
335        args,
336        environ: None,
337        ns,
338        job: job_default().duplicate(zx::Rights::SAME_RIGHTS)?,
339        options: zx::ProcessOptions::empty(),
340        config: None,
341    })
342    .await?;
343    Ok(Arc::new(component))
344}
345
346#[cfg(test)]
347mod tests {
348    use super::*;
349    use fidl_fuchsia_test::Status;
350    use maplit::hashmap;
351
352    #[test]
353    fn test_ordering_by_enum() {
354        let expected_events = vec![
355            ListenerEvent::start_test("a"),
356            ListenerEvent::finish_test(
357                "a",
358                TestResult { status: Some(Status::Passed), ..Default::default() },
359            ),
360            ListenerEvent::finish_all_test(),
361        ];
362
363        let mut events = expected_events.clone();
364        events.reverse();
365
366        assert_ne!(events, expected_events);
367        events.sort();
368        assert_eq!(events, expected_events);
369    }
370
371    #[test]
372    fn test_ordering_by_test_name() {
373        let mut events = vec![
374            ListenerEvent::start_test("b"),
375            ListenerEvent::start_test("a"),
376            ListenerEvent::finish_test(
377                "a",
378                TestResult { status: Some(Status::Passed), ..Default::default() },
379            ),
380            ListenerEvent::start_test("c"),
381            ListenerEvent::finish_test(
382                "b",
383                TestResult { status: Some(Status::Passed), ..Default::default() },
384            ),
385            ListenerEvent::finish_test(
386                "c",
387                TestResult { status: Some(Status::Passed), ..Default::default() },
388            ),
389            ListenerEvent::finish_all_test(),
390        ];
391
392        let expected_events = vec![
393            ListenerEvent::start_test("a"),
394            ListenerEvent::finish_test(
395                "a",
396                TestResult { status: Some(Status::Passed), ..Default::default() },
397            ),
398            ListenerEvent::start_test("b"),
399            ListenerEvent::finish_test(
400                "b",
401                TestResult { status: Some(Status::Passed), ..Default::default() },
402            ),
403            ListenerEvent::start_test("c"),
404            ListenerEvent::finish_test(
405                "c",
406                TestResult { status: Some(Status::Passed), ..Default::default() },
407            ),
408            ListenerEvent::finish_all_test(),
409        ];
410        events.sort();
411        assert_eq!(events, expected_events);
412    }
413
414    #[test]
415    fn line_buffer_std_message_incomplete_line() {
416        let mut buf = HashMap::new();
417        buf.insert("test".to_string(), "some_prev_text".to_string());
418        let strings = line_buffer_std_message("test", "a \nb\nc\nd".into(), false, &mut buf);
419        assert_eq!(strings, vec!["some_prev_texta ".to_owned(), "b".to_owned(), "c".to_owned()]);
420        assert_eq!(buf, hashmap! {"test".to_string() => "d".to_string()});
421    }
422
423    #[test]
424    fn line_buffer_std_message_complete_line() {
425        let mut buf = HashMap::new();
426        buf.insert("test".to_string(), "some_prev_text".to_string());
427        let strings = line_buffer_std_message("test", "a \nb\nc\n".into(), false, &mut buf);
428        assert_eq!(strings, vec!["some_prev_texta ".to_owned(), "b".to_owned(), "c".to_owned()]);
429        assert_eq!(buf.len(), 0);
430
431        // test when initial buf is empty
432        let strings = line_buffer_std_message("test", "d \ne\nf\n".into(), false, &mut buf);
433        assert_eq!(strings, vec!["d ".to_owned(), "e".to_owned(), "f".to_owned()]);
434        assert_eq!(buf.len(), 0);
435    }
436}