1use crate::cancel::{Cancelled, NamedFutureExt, OrCancel};
6use crate::connector::SuiteRunnerConnector;
7use crate::diagnostics::{self, LogDisplayConfiguration};
8use crate::outcome::{Outcome, RunTestSuiteError};
9use crate::output::{self, RunReporter, Timestamp};
10use crate::params::{RunParams, TestParams, TimeoutBehavior};
11use crate::running_suite::{RunningSuite, WaitForStartArgs, run_suite_and_collect_logs};
12use diagnostics_data::LogTextDisplayOptions;
13use fidl_fuchsia_test_manager::{self as ftest_manager, SuiteRunnerProxy};
14use futures::prelude::*;
15use log::warn;
16use std::io::Write;
17use std::path::PathBuf;
18
19struct RunState<'a> {
20 run_params: &'a RunParams,
21 final_outcome: Option<Outcome>,
22 failed_suites: u32,
23 timeout_occurred: bool,
24 cancel_occurred: bool,
25 internal_error_occurred: bool,
26}
27
28impl<'a> RunState<'a> {
29 fn new(run_params: &'a RunParams) -> Self {
30 Self {
31 run_params,
32 final_outcome: None,
33 failed_suites: 0,
34 timeout_occurred: false,
35 cancel_occurred: false,
36 internal_error_occurred: false,
37 }
38 }
39
40 fn cancel_run(&mut self, final_outcome: Outcome) {
41 self.final_outcome = Some(final_outcome);
42 self.cancel_occurred = true;
43 }
44
45 fn record_next_outcome(&mut self, next_outcome: Outcome) {
46 if next_outcome != Outcome::Passed {
47 self.failed_suites += 1;
48 }
49 match &next_outcome {
50 Outcome::Timedout => self.timeout_occurred = true,
51 Outcome::Cancelled => self.cancel_occurred = true,
52 Outcome::Error { origin } if origin.is_internal_error() => {
53 self.internal_error_occurred = true;
54 }
55 Outcome::Passed
56 | Outcome::Failed
57 | Outcome::Inconclusive
58 | Outcome::DidNotFinish
59 | Outcome::Error { .. } => (),
60 }
61
62 self.final_outcome = match (self.final_outcome.take(), next_outcome) {
63 (None, first_outcome) => Some(first_outcome),
64 (Some(outcome), Outcome::Passed) => Some(outcome),
65 (Some(_), failing_outcome) => Some(failing_outcome),
66 };
67 }
68
69 fn should_stop_run(&mut self) -> bool {
70 let stop_due_to_timeout = self.run_params.timeout_behavior
71 == TimeoutBehavior::TerminateRemaining
72 && self.timeout_occurred;
73 let stop_due_to_failures = match self.run_params.stop_after_failures.as_ref() {
74 Some(threshold) => self.failed_suites >= threshold.get(),
75 None => false,
76 };
77 stop_due_to_timeout
78 || stop_due_to_failures
79 || self.cancel_occurred
80 || self.internal_error_occurred
81 }
82
83 fn final_outcome(self) -> Outcome {
84 self.final_outcome.unwrap_or(Outcome::Passed)
85 }
86}
87
88async fn run_test_chunk<'a, F: 'a + Future<Output = ()> + Unpin>(
92 runner_proxy: SuiteRunnerProxy,
93 test_params: TestParams,
94 run_state: &'a mut RunState<'_>,
95 run_params: &'a RunParams,
96 run_reporter: &'a RunReporter,
97 cancel_fut: F,
98) -> Result<(), RunTestSuiteError> {
99 let timeout = test_params
100 .timeout_seconds
101 .map(|seconds| std::time::Duration::from_secs(seconds.get() as u64));
102
103 let mut combined_log_interest = run_params.min_severity_logs.clone();
106 combined_log_interest.extend(test_params.min_severity_logs.iter().cloned());
107
108 let mut run_options = fidl_fuchsia_test_manager::RunSuiteOptions {
109 max_concurrent_test_case_runs: test_params.parallel,
110 arguments: Some(test_params.test_args),
111 timeout: timeout.map(|duration| duration.as_nanos() as i64),
112 run_disabled_tests: Some(test_params.also_run_disabled_tests),
113 test_case_filters: test_params.test_filters,
114 break_on_failure: Some(test_params.break_on_failure),
115 logs_iterator_type: Some(
116 run_params.log_protocol.unwrap_or_else(diagnostics::get_logs_iterator_type),
117 ),
118 log_interest: Some(combined_log_interest),
119 no_exception_channel: Some(test_params.no_exception_channel),
120 ..Default::default()
121 };
122 let suite = run_reporter.new_suite(&test_params.test_url)?;
123 suite.set_tags(test_params.tags);
124 if let Some(realm) = test_params.realm.as_ref() {
125 run_options.realm_options = Some(fidl_fuchsia_test_manager::RealmOptions {
126 realm: Some(realm.get_realm_client()?),
127 offers: Some(realm.offers()),
128 test_collection: Some(realm.collection().to_string()),
129 ..Default::default()
130 });
131 }
132 let (suite_controller, suite_server_end) =
133 fidl::endpoints::create_proxy::<ftest_manager::SuiteControllerMarker>();
134 let suite_start_fut = RunningSuite::wait_for_start(WaitForStartArgs {
135 proxy: suite_controller,
136 max_severity_logs: test_params.max_severity_logs,
137 timeout,
138 timeout_grace: std::time::Duration::from_secs(run_params.timeout_grace_seconds as u64),
139 max_pipelined: None,
140 no_cases_equals_success: test_params.no_cases_equals_success,
141 });
142
143 runner_proxy.run(&test_params.test_url, run_options, suite_server_end)?;
144
145 let cancel_fut = cancel_fut.shared();
146
147 let handle_suite_fut = async move {
148 'block: {
150 let suite_stop_fut = cancel_fut.clone().map(|_| Outcome::Cancelled);
151
152 let running_suite =
153 match suite_start_fut.named("suite_start").or_cancelled(suite_stop_fut).await {
154 Ok(running_suite) => running_suite,
155 Err(Cancelled(final_outcome)) => {
156 run_state.cancel_run(final_outcome);
157 break 'block;
158 }
159 };
160
161 let log_display = LogDisplayConfiguration {
162 interest: run_params.min_severity_logs.clone(),
163 text_options: LogTextDisplayOptions {
164 show_full_moniker: run_params.show_full_moniker,
165 ..Default::default()
166 },
167 };
168
169 let result =
170 run_suite_and_collect_logs(running_suite, &suite, log_display, cancel_fut.clone())
171 .await;
172 let suite_outcome = result.unwrap_or_else(|err| Outcome::error(err));
173 suite.finished()?;
175 run_state.record_next_outcome(suite_outcome);
176 if run_state.should_stop_run() {
177 break 'block;
178 }
179 }
180 Result::<_, RunTestSuiteError>::Ok(())
181 };
182
183 handle_suite_fut.boxed_local().await.map_err(|e| e.into())
184}
185
186async fn run_tests<'a, F: 'a + Future<Output = ()> + Unpin>(
187 connector: impl SuiteRunnerConnector,
188 test_params: TestParams,
189 run_params: RunParams,
190 run_reporter: &'a RunReporter,
191 cancel_fut: F,
192) -> Result<Outcome, RunTestSuiteError> {
193 let mut run_state = RunState::new(&run_params);
194 let cancel_fut = cancel_fut.shared();
195 match run_state.should_stop_run() {
196 true => {
197 let suite_reporter = run_reporter.new_suite(&test_params.test_url)?;
200 suite_reporter.set_tags(test_params.tags);
201 suite_reporter.finished()?;
202 }
203 false => {
204 let runner_proxy = connector.connect().await?;
205 run_test_chunk(
206 runner_proxy,
207 test_params,
208 &mut run_state,
209 &run_params,
210 run_reporter,
211 cancel_fut.clone(),
212 )
213 .await?;
214 }
215 }
216
217 Ok(run_state.final_outcome())
218}
219
220pub async fn run_test_and_get_outcome<F>(
231 connector: impl SuiteRunnerConnector,
232 test_params: TestParams,
233 run_params: RunParams,
234 run_reporter: RunReporter,
235 cancel_fut: F,
236) -> Outcome
237where
238 F: Future<Output = ()>,
239{
240 match run_reporter.started(Timestamp::Unknown) {
241 Ok(()) => (),
242 Err(e) => return Outcome::error(e),
243 }
244 let test_outcome = match run_tests(
245 connector,
246 test_params,
247 run_params,
248 &run_reporter,
249 cancel_fut.boxed_local(),
250 )
251 .await
252 {
253 Ok(s) => s,
254 Err(e) => {
255 return Outcome::error(e);
256 }
257 };
258
259 let report_result = match run_reporter.stopped(&test_outcome.clone().into(), Timestamp::Unknown)
260 {
261 Ok(()) => run_reporter.finished(),
262 Err(e) => Err(e),
263 };
264 if let Err(e) = report_result {
265 warn!("Failed to record results: {:?}", e);
266 }
267
268 test_outcome
269}
270
271pub struct DirectoryReporterOptions {
272 pub root_path: PathBuf,
274}
275
276pub fn create_reporter<W: 'static + Write + Send + Sync>(
278 filter_ansi: bool,
279 dir: Option<DirectoryReporterOptions>,
280 writer: W,
281) -> Result<output::RunReporter, anyhow::Error> {
282 let stdout_reporter = output::ShellReporter::new(writer);
283 let dir_reporter = dir
284 .map(|dir| {
285 output::DirectoryWithStdoutReporter::new(dir.root_path, output::SchemaVersion::V1)
286 })
287 .transpose()?;
288 let reporter = match (dir_reporter, filter_ansi) {
289 (Some(dir_reporter), false) => output::RunReporter::new(output::MultiplexedReporter::new(
290 stdout_reporter,
291 dir_reporter,
292 )),
293 (Some(dir_reporter), true) => output::RunReporter::new_ansi_filtered(
294 output::MultiplexedReporter::new(stdout_reporter, dir_reporter),
295 ),
296 (None, false) => output::RunReporter::new(stdout_reporter),
297 (None, true) => output::RunReporter::new_ansi_filtered(stdout_reporter),
298 };
299 Ok(reporter)
300}
301
302#[cfg(test)]
303mod test {
304 use super::*;
305 use crate::connector::SingleRunConnector;
306 use crate::output::{EntityId, InMemoryReporter};
307 use assert_matches::assert_matches;
308 use fidl::endpoints::{Proxy, create_proxy_and_stream};
309 use fidl_fuchsia_test_manager as ftest_manager;
310 use futures::future::join;
311 use futures::stream::futures_unordered::FuturesUnordered;
312 use maplit::hashmap;
313 use std::collections::HashMap;
314 #[cfg(target_os = "fuchsia")]
315 use {
316 fidl_fuchsia_io as fio, futures::future::join3, vfs::file::vmo::read_only,
317 vfs::pseudo_directory, zx,
318 };
319
320 async fn fake_running_all_suites(
323 mut stream: ftest_manager::SuiteRunnerRequestStream,
324 mut suite_events: HashMap<&str, Vec<ftest_manager::Event>>,
325 ) {
326 let mut suite_streams = vec![];
327
328 if let Ok(Some(req)) = stream.try_next().await {
329 match req {
330 ftest_manager::SuiteRunnerRequest::Run { test_suite_url, controller, .. } => {
331 let events = suite_events
332 .remove(test_suite_url.as_str())
333 .expect("Got a request for an unexpected test URL");
334 suite_streams.push((controller.into_stream(), events));
335 }
336 ftest_manager::SuiteRunnerRequest::_UnknownMethod { ordinal, .. } => {
337 panic!("Not expecting unknown request: {}", ordinal)
338 }
339 }
340 }
341 assert!(suite_events.is_empty(), "Expected AddSuite to be called for all specified suites");
342
343 let mut suite_streams = suite_streams
345 .into_iter()
346 .map(|(mut stream, events)| {
347 async move {
348 let mut maybe_events = Some(events);
349 while let Ok(Some(req)) = stream.try_next().await {
350 match req {
351 ftest_manager::SuiteControllerRequest::WatchEvents {
352 responder,
353 ..
354 } => {
355 let send_events = maybe_events.take().unwrap_or(vec![]);
356 let _ = responder.send(Ok(send_events));
357 }
358 _ => {
359 }
361 }
362 }
363 }
364 })
365 .collect::<FuturesUnordered<_>>();
366
367 async move { while let Some(_) = suite_streams.next().await {} }.await;
368 }
369
370 struct ParamsForRunTests {
371 runner_proxy: ftest_manager::SuiteRunnerProxy,
372 test_params: TestParams,
373 run_reporter: RunReporter,
374 }
375
376 fn create_empty_suite_events() -> Vec<ftest_manager::Event> {
377 vec![
378 ftest_manager::Event {
379 timestamp: Some(1000),
380 details: Some(ftest_manager::EventDetails::SuiteStarted(
381 ftest_manager::SuiteStartedEventDetails { ..Default::default() },
382 )),
383 ..Default::default()
384 },
385 ftest_manager::Event {
386 timestamp: Some(2000),
387 details: Some(ftest_manager::EventDetails::SuiteStopped(
388 ftest_manager::SuiteStoppedEventDetails {
389 result: Some(ftest_manager::SuiteResult::Finished),
390 ..Default::default()
391 },
392 )),
393 ..Default::default()
394 },
395 ]
396 }
397
398 async fn call_run_tests(params: ParamsForRunTests) -> Outcome {
399 run_test_and_get_outcome(
400 SingleRunConnector::new(params.runner_proxy),
401 params.test_params,
402 RunParams {
403 timeout_behavior: TimeoutBehavior::Continue,
404 timeout_grace_seconds: 0,
405 stop_after_failures: None,
406 accumulate_debug_data: false,
407 log_protocol: None,
408 min_severity_logs: vec![],
409 show_full_moniker: false,
410 },
411 params.run_reporter,
412 futures::future::pending(),
413 )
414 .await
415 }
416
417 #[fuchsia::test]
418 async fn single_run_no_events() {
419 let (runner_proxy, suite_runner_stream) =
420 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
421
422 let reporter = InMemoryReporter::new();
423 let run_reporter = RunReporter::new(reporter.clone());
424 let run_fut = call_run_tests(ParamsForRunTests {
425 runner_proxy,
426 test_params: TestParams {
427 test_url: "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm".to_string(),
428 ..TestParams::default()
429 },
430 run_reporter,
431 });
432 let fake_fut = fake_running_all_suites(
433 suite_runner_stream,
434 hashmap! {
435 "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm" => create_empty_suite_events()
436 },
437 );
438
439 assert_eq!(join(run_fut, fake_fut).await.0, Outcome::Passed,);
440
441 let reports = reporter.get_reports();
442 assert_eq!(2usize, reports.len());
443 assert!(reports[0].report.artifacts.is_empty());
444 assert!(reports[0].report.directories.is_empty());
445 assert!(reports[1].report.artifacts.is_empty());
446 assert!(reports[1].report.directories.is_empty());
447 }
448
449 #[cfg(target_os = "fuchsia")]
450 #[fuchsia::test]
451 async fn single_run_custom_directory() {
452 let (runner_proxy, suite_runner_stream) =
453 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
454
455 let reporter = InMemoryReporter::new();
456 let run_reporter = RunReporter::new(reporter.clone());
457 let run_fut = call_run_tests(ParamsForRunTests {
458 runner_proxy,
459 test_params: TestParams {
460 test_url: "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm".to_string(),
461 ..TestParams::default()
462 },
463 run_reporter,
464 });
465
466 let dir = pseudo_directory! {
467 "test_file.txt" => read_only("Hello, World!"),
468 };
469
470 let directory_proxy = vfs::directory::serve(dir, fio::PERM_READABLE | fio::PERM_WRITABLE);
471
472 let directory_client =
473 fidl::endpoints::ClientEnd::new(directory_proxy.into_channel().unwrap().into());
474
475 let (_pair_1, pair_2) = zx::EventPair::create();
476
477 let events = vec![
478 ftest_manager::Event {
479 timestamp: Some(1000),
480 details: Some(ftest_manager::EventDetails::SuiteStarted(
481 ftest_manager::SuiteStartedEventDetails { ..Default::default() },
482 )),
483 ..Default::default()
484 },
485 ftest_manager::Event {
486 details: Some(ftest_manager::EventDetails::SuiteArtifactGenerated(
487 ftest_manager::SuiteArtifactGeneratedEventDetails {
488 artifact: Some(ftest_manager::Artifact::Custom(
489 ftest_manager::CustomArtifact {
490 directory_and_token: Some(ftest_manager::DirectoryAndToken {
491 directory: directory_client,
492 token: pair_2,
493 }),
494 ..Default::default()
495 },
496 )),
497 ..Default::default()
498 },
499 )),
500 ..Default::default()
501 },
502 ftest_manager::Event {
503 timestamp: Some(2000),
504 details: Some(ftest_manager::EventDetails::SuiteStopped(
505 ftest_manager::SuiteStoppedEventDetails {
506 result: Some(ftest_manager::SuiteResult::Finished),
507 ..Default::default()
508 },
509 )),
510 ..Default::default()
511 },
512 ];
513
514 let fake_fut = fake_running_all_suites(
515 suite_runner_stream,
516 hashmap! {
517 "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm" => events
518 },
519 );
520
521 assert_eq!(join(run_fut, fake_fut).await.0, Outcome::Passed,);
522
523 let reports = reporter.get_reports();
524 assert_eq!(2usize, reports.len());
525 let run = reports.iter().find(|e| e.id == EntityId::Suite).expect("find run report");
526 assert_eq!(1usize, run.report.directories.len());
527 let dir = &run.report.directories[0];
528 let files = dir.1.files.lock();
529 assert_eq!(1usize, files.len());
530 let (name, file) = &files[0];
531 assert_eq!(name.to_string_lossy(), "test_file.txt".to_string());
532 assert_eq!(file.get_contents(), b"Hello, World!");
533 }
534
535 #[fuchsia::test]
536 async fn record_output_after_internal_error() {
537 let (runner_proxy, suite_runner_stream) =
538 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
539
540 let reporter = InMemoryReporter::new();
541 let run_reporter = RunReporter::new(reporter.clone());
542 let run_fut = call_run_tests(ParamsForRunTests {
543 runner_proxy,
544 test_params: TestParams {
545 test_url: "fuchsia-pkg://fuchsia.com/invalid#meta/invalid.cm".to_string(),
546 ..TestParams::default()
547 },
548 run_reporter,
549 });
550
551 let fake_fut = fake_running_all_suites(
552 suite_runner_stream,
553 hashmap! {
554 "fuchsia-pkg://fuchsia.com/invalid#meta/invalid.cm" => vec![
556 ftest_manager::Event {
557 timestamp: Some(1000),
558 details: Some(ftest_manager::EventDetails::SuiteStarted(ftest_manager::SuiteStartedEventDetails {..Default::default()})),
559 ..Default::default()
560 },
561 ftest_manager::Event {
562 timestamp: Some(2000),
563 details: Some(ftest_manager::EventDetails::SuiteStopped(ftest_manager::SuiteStoppedEventDetails {
564 result: Some(ftest_manager::SuiteResult::InternalError),
565 ..Default::default()
566 },
567 )),
568 ..Default::default()
569 },
570 ],
571 },
572 );
573
574 assert_matches!(join(run_fut, fake_fut).await.0, Outcome::Error { .. });
575
576 let reports = reporter.get_reports();
577 assert_eq!(2usize, reports.len());
578 let invalid_suite = reports
579 .iter()
580 .find(|e| e.report.name == "fuchsia-pkg://fuchsia.com/invalid#meta/invalid.cm")
581 .expect("find run report");
582 assert_eq!(invalid_suite.report.outcome, Some(output::ReportedOutcome::Error));
583 assert!(invalid_suite.report.is_finished);
584
585 let run = reports.iter().find(|e| e.id == EntityId::TestRun).expect("find run report");
587 assert_eq!(run.report.outcome, Some(output::ReportedOutcome::Error));
588 assert!(run.report.is_finished);
589 assert!(run.report.started_time.is_some());
590 }
591
592 #[cfg(target_os = "fuchsia")]
593 #[fuchsia::test]
594 async fn single_run_debug_data() {
595 let (runner_proxy, suite_runner_stream) =
596 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
597
598 let reporter = InMemoryReporter::new();
599 let run_reporter = RunReporter::new(reporter.clone());
600 let run_fut = call_run_tests(ParamsForRunTests {
601 runner_proxy,
602 test_params: TestParams {
603 test_url: "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm".to_string(),
604 ..TestParams::default()
605 },
606 run_reporter,
607 });
608
609 let (debug_client, debug_service) =
610 fidl::endpoints::create_endpoints::<ftest_manager::DebugDataIteratorMarker>();
611 let debug_data_fut = async move {
612 let (client, server) = zx::Socket::create_stream();
613 let mut compressor = zstd::bulk::Compressor::new(0).unwrap();
614 let bytes = compressor.compress(b"Not a real profile").unwrap();
615 let _ = server.write(bytes.as_slice()).unwrap();
616 let mut service = debug_service.into_stream();
617 let mut data = vec![ftest_manager::DebugData {
618 name: Some("test_file.profraw".to_string()),
619 socket: Some(client.into()),
620 ..Default::default()
621 }];
622 drop(server);
623 while let Ok(Some(request)) = service.try_next().await {
624 match request {
625 ftest_manager::DebugDataIteratorRequest::GetNext { .. } => {
626 panic!("Not Implemented");
627 }
628 ftest_manager::DebugDataIteratorRequest::GetNextCompressed {
629 responder,
630 ..
631 } => {
632 let _ = responder.send(std::mem::take(&mut data));
633 }
634 }
635 }
636 };
637
638 let events = vec![
639 ftest_manager::Event {
640 timestamp: Some(1000),
641 details: Some(ftest_manager::EventDetails::SuiteStarted(
642 ftest_manager::SuiteStartedEventDetails { ..Default::default() },
643 )),
644 ..Default::default()
645 },
646 ftest_manager::Event {
647 details: Some(ftest_manager::EventDetails::SuiteArtifactGenerated(
648 ftest_manager::SuiteArtifactGeneratedEventDetails {
649 artifact: Some(ftest_manager::Artifact::DebugData(debug_client)),
650 ..Default::default()
651 },
652 )),
653 ..Default::default()
654 },
655 ftest_manager::Event {
656 timestamp: Some(2000),
657 details: Some(ftest_manager::EventDetails::SuiteStopped(
658 ftest_manager::SuiteStoppedEventDetails {
659 result: Some(ftest_manager::SuiteResult::Finished),
660 ..Default::default()
661 },
662 )),
663 ..Default::default()
664 },
665 ];
666
667 let fake_fut = fake_running_all_suites(
668 suite_runner_stream,
669 hashmap! {
670 "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm" => events,
671 },
672 );
673
674 assert_eq!(join3(run_fut, debug_data_fut, fake_fut).await.0, Outcome::Passed);
675
676 let reports = reporter.get_reports();
677 assert_eq!(2usize, reports.len());
678 let run = reports.iter().find(|e| e.id == EntityId::Suite).expect("find run report");
679 assert_eq!(1usize, run.report.directories.len());
680 let dir = &run.report.directories[0];
681 let files = dir.1.files.lock();
682 assert_eq!(1usize, files.len());
683 let (name, file) = &files[0];
684 assert_eq!(name.to_string_lossy(), "test_file.profraw".to_string());
685 assert_eq!(file.get_contents(), b"Not a real profile");
686 }
687}