1use crate::cancel::{Cancelled, NamedFutureExt, OrCancel};
6use crate::connector::SuiteRunnerConnector;
7use crate::diagnostics::{self, LogDisplayConfiguration};
8use crate::outcome::{Outcome, RunTestSuiteError};
9use crate::output::{self, RunReporter, Timestamp};
10use crate::params::{RunParams, TestParams, TimeoutBehavior};
11use crate::running_suite::{run_suite_and_collect_logs, RunningSuite, WaitForStartArgs};
12use diagnostics_data::LogTextDisplayOptions;
13use fidl_fuchsia_test_manager::{self as ftest_manager, SuiteRunnerProxy};
14use futures::prelude::*;
15use log::warn;
16use std::io::Write;
17use std::path::PathBuf;
18
19struct RunState<'a> {
20 run_params: &'a RunParams,
21 final_outcome: Option<Outcome>,
22 failed_suites: u32,
23 timeout_occurred: bool,
24 cancel_occurred: bool,
25 internal_error_occurred: bool,
26}
27
28impl<'a> RunState<'a> {
29 fn new(run_params: &'a RunParams) -> Self {
30 Self {
31 run_params,
32 final_outcome: None,
33 failed_suites: 0,
34 timeout_occurred: false,
35 cancel_occurred: false,
36 internal_error_occurred: false,
37 }
38 }
39
40 fn cancel_run(&mut self, final_outcome: Outcome) {
41 self.final_outcome = Some(final_outcome);
42 self.cancel_occurred = true;
43 }
44
45 fn record_next_outcome(&mut self, next_outcome: Outcome) {
46 if next_outcome != Outcome::Passed {
47 self.failed_suites += 1;
48 }
49 match &next_outcome {
50 Outcome::Timedout => self.timeout_occurred = true,
51 Outcome::Cancelled => self.cancel_occurred = true,
52 Outcome::Error { origin } if origin.is_internal_error() => {
53 self.internal_error_occurred = true;
54 }
55 Outcome::Passed
56 | Outcome::Failed
57 | Outcome::Inconclusive
58 | Outcome::DidNotFinish
59 | Outcome::Error { .. } => (),
60 }
61
62 self.final_outcome = match (self.final_outcome.take(), next_outcome) {
63 (None, first_outcome) => Some(first_outcome),
64 (Some(outcome), Outcome::Passed) => Some(outcome),
65 (Some(_), failing_outcome) => Some(failing_outcome),
66 };
67 }
68
69 fn should_stop_run(&mut self) -> bool {
70 let stop_due_to_timeout = self.run_params.timeout_behavior
71 == TimeoutBehavior::TerminateRemaining
72 && self.timeout_occurred;
73 let stop_due_to_failures = match self.run_params.stop_after_failures.as_ref() {
74 Some(threshold) => self.failed_suites >= threshold.get(),
75 None => false,
76 };
77 stop_due_to_timeout
78 || stop_due_to_failures
79 || self.cancel_occurred
80 || self.internal_error_occurred
81 }
82
83 fn final_outcome(self) -> Outcome {
84 self.final_outcome.unwrap_or(Outcome::Passed)
85 }
86}
87
88async fn run_test_chunk<'a, F: 'a + Future<Output = ()> + Unpin>(
92 runner_proxy: SuiteRunnerProxy,
93 test_params: TestParams,
94 run_state: &'a mut RunState<'_>,
95 run_params: &'a RunParams,
96 run_reporter: &'a RunReporter,
97 cancel_fut: F,
98) -> Result<(), RunTestSuiteError> {
99 let timeout = test_params
100 .timeout_seconds
101 .map(|seconds| std::time::Duration::from_secs(seconds.get() as u64));
102
103 let mut combined_log_interest = run_params.min_severity_logs.clone();
106 combined_log_interest.extend(test_params.min_severity_logs.iter().cloned());
107
108 let mut run_options = fidl_fuchsia_test_manager::RunSuiteOptions {
109 max_concurrent_test_case_runs: test_params.parallel,
110 arguments: Some(test_params.test_args),
111 run_disabled_tests: Some(test_params.also_run_disabled_tests),
112 test_case_filters: test_params.test_filters,
113 break_on_failure: Some(test_params.break_on_failure),
114 logs_iterator_type: Some(
115 run_params.log_protocol.unwrap_or_else(diagnostics::get_logs_iterator_type),
116 ),
117 log_interest: Some(combined_log_interest),
118 no_exception_channel: Some(test_params.no_exception_channel),
119 ..Default::default()
120 };
121 let suite = run_reporter.new_suite(&test_params.test_url)?;
122 suite.set_tags(test_params.tags);
123 if let Some(realm) = test_params.realm.as_ref() {
124 run_options.realm_options = Some(fidl_fuchsia_test_manager::RealmOptions {
125 realm: Some(realm.get_realm_client()?),
126 offers: Some(realm.offers()),
127 test_collection: Some(realm.collection().to_string()),
128 ..Default::default()
129 });
130 }
131 let (suite_controller, suite_server_end) =
132 fidl::endpoints::create_proxy::<ftest_manager::SuiteControllerMarker>();
133 let suite_start_fut = RunningSuite::wait_for_start(WaitForStartArgs {
134 proxy: suite_controller,
135 max_severity_logs: test_params.max_severity_logs,
136 timeout,
137 timeout_grace: std::time::Duration::from_secs(run_params.timeout_grace_seconds as u64),
138 max_pipelined: None,
139 no_cases_equals_success: test_params.no_cases_equals_success,
140 });
141
142 runner_proxy.run(&test_params.test_url, run_options, suite_server_end)?;
143
144 let cancel_fut = cancel_fut.shared();
145
146 let handle_suite_fut = async move {
147 'block: {
149 let suite_stop_fut = cancel_fut.clone().map(|_| Outcome::Cancelled);
150
151 let running_suite =
152 match suite_start_fut.named("suite_start").or_cancelled(suite_stop_fut).await {
153 Ok(running_suite) => running_suite,
154 Err(Cancelled(final_outcome)) => {
155 run_state.cancel_run(final_outcome);
156 break 'block;
157 }
158 };
159
160 let log_display = LogDisplayConfiguration {
161 interest: run_params.min_severity_logs.clone(),
162 text_options: LogTextDisplayOptions {
163 show_full_moniker: run_params.show_full_moniker,
164 ..Default::default()
165 },
166 };
167
168 let result =
169 run_suite_and_collect_logs(running_suite, &suite, log_display, cancel_fut.clone())
170 .await;
171 let suite_outcome = result.unwrap_or_else(|err| Outcome::error(err));
172 suite.finished()?;
174 run_state.record_next_outcome(suite_outcome);
175 if run_state.should_stop_run() {
176 break 'block;
177 }
178 }
179 Result::<_, RunTestSuiteError>::Ok(())
180 };
181
182 handle_suite_fut.boxed_local().await.map_err(|e| e.into())
183}
184
185async fn run_tests<'a, F: 'a + Future<Output = ()> + Unpin>(
186 connector: impl SuiteRunnerConnector,
187 test_params: TestParams,
188 run_params: RunParams,
189 run_reporter: &'a RunReporter,
190 cancel_fut: F,
191) -> Result<Outcome, RunTestSuiteError> {
192 let mut run_state = RunState::new(&run_params);
193 let cancel_fut = cancel_fut.shared();
194 match run_state.should_stop_run() {
195 true => {
196 let suite_reporter = run_reporter.new_suite(&test_params.test_url)?;
199 suite_reporter.set_tags(test_params.tags);
200 suite_reporter.finished()?;
201 }
202 false => {
203 let runner_proxy = connector.connect().await?;
204 run_test_chunk(
205 runner_proxy,
206 test_params,
207 &mut run_state,
208 &run_params,
209 run_reporter,
210 cancel_fut.clone(),
211 )
212 .await?;
213 }
214 }
215
216 Ok(run_state.final_outcome())
217}
218
219pub async fn run_test_and_get_outcome<F>(
230 connector: impl SuiteRunnerConnector,
231 test_params: TestParams,
232 run_params: RunParams,
233 run_reporter: RunReporter,
234 cancel_fut: F,
235) -> Outcome
236where
237 F: Future<Output = ()>,
238{
239 match run_reporter.started(Timestamp::Unknown) {
240 Ok(()) => (),
241 Err(e) => return Outcome::error(e),
242 }
243 let test_outcome = match run_tests(
244 connector,
245 test_params,
246 run_params,
247 &run_reporter,
248 cancel_fut.boxed_local(),
249 )
250 .await
251 {
252 Ok(s) => s,
253 Err(e) => {
254 return Outcome::error(e);
255 }
256 };
257
258 let report_result = match run_reporter.stopped(&test_outcome.clone().into(), Timestamp::Unknown)
259 {
260 Ok(()) => run_reporter.finished(),
261 Err(e) => Err(e),
262 };
263 if let Err(e) = report_result {
264 warn!("Failed to record results: {:?}", e);
265 }
266
267 test_outcome
268}
269
270pub struct DirectoryReporterOptions {
271 pub root_path: PathBuf,
273}
274
275pub fn create_reporter<W: 'static + Write + Send + Sync>(
277 filter_ansi: bool,
278 dir: Option<DirectoryReporterOptions>,
279 writer: W,
280) -> Result<output::RunReporter, anyhow::Error> {
281 let stdout_reporter = output::ShellReporter::new(writer);
282 let dir_reporter = dir
283 .map(|dir| {
284 output::DirectoryWithStdoutReporter::new(dir.root_path, output::SchemaVersion::V1)
285 })
286 .transpose()?;
287 let reporter = match (dir_reporter, filter_ansi) {
288 (Some(dir_reporter), false) => output::RunReporter::new(output::MultiplexedReporter::new(
289 stdout_reporter,
290 dir_reporter,
291 )),
292 (Some(dir_reporter), true) => output::RunReporter::new_ansi_filtered(
293 output::MultiplexedReporter::new(stdout_reporter, dir_reporter),
294 ),
295 (None, false) => output::RunReporter::new(stdout_reporter),
296 (None, true) => output::RunReporter::new_ansi_filtered(stdout_reporter),
297 };
298 Ok(reporter)
299}
300
301#[cfg(test)]
302mod test {
303 use super::*;
304 use crate::connector::SingleRunConnector;
305 use crate::output::{EntityId, InMemoryReporter};
306 use assert_matches::assert_matches;
307 use fidl::endpoints::{create_proxy_and_stream, Proxy};
308 use fidl_fuchsia_test_manager as ftest_manager;
309 use futures::future::join;
310 use futures::stream::futures_unordered::FuturesUnordered;
311 use maplit::hashmap;
312 use std::collections::HashMap;
313 #[cfg(target_os = "fuchsia")]
314 use {
315 fidl_fuchsia_io as fio, futures::future::join3, vfs::file::vmo::read_only,
316 vfs::pseudo_directory, zx,
317 };
318
319 async fn fake_running_all_suites(
322 mut stream: ftest_manager::SuiteRunnerRequestStream,
323 mut suite_events: HashMap<&str, Vec<ftest_manager::Event>>,
324 ) {
325 let mut suite_streams = vec![];
326
327 if let Ok(Some(req)) = stream.try_next().await {
328 match req {
329 ftest_manager::SuiteRunnerRequest::Run { test_suite_url, controller, .. } => {
330 let events = suite_events
331 .remove(test_suite_url.as_str())
332 .expect("Got a request for an unexpected test URL");
333 suite_streams.push((controller.into_stream(), events));
334 }
335 ftest_manager::SuiteRunnerRequest::_UnknownMethod { ordinal, .. } => {
336 panic!("Not expecting unknown request: {}", ordinal)
337 }
338 }
339 }
340 assert!(suite_events.is_empty(), "Expected AddSuite to be called for all specified suites");
341
342 let mut suite_streams = suite_streams
344 .into_iter()
345 .map(|(mut stream, events)| {
346 async move {
347 let mut maybe_events = Some(events);
348 while let Ok(Some(req)) = stream.try_next().await {
349 match req {
350 ftest_manager::SuiteControllerRequest::WatchEvents {
351 responder,
352 ..
353 } => {
354 let send_events = maybe_events.take().unwrap_or(vec![]);
355 let _ = responder.send(Ok(send_events));
356 }
357 _ => {
358 }
360 }
361 }
362 }
363 })
364 .collect::<FuturesUnordered<_>>();
365
366 async move { while let Some(_) = suite_streams.next().await {} }.await;
367 }
368
369 struct ParamsForRunTests {
370 runner_proxy: ftest_manager::SuiteRunnerProxy,
371 test_params: TestParams,
372 run_reporter: RunReporter,
373 }
374
375 fn create_empty_suite_events() -> Vec<ftest_manager::Event> {
376 vec![
377 ftest_manager::Event {
378 timestamp: Some(1000),
379 details: Some(ftest_manager::EventDetails::SuiteStarted(
380 ftest_manager::SuiteStartedEventDetails { ..Default::default() },
381 )),
382 ..Default::default()
383 },
384 ftest_manager::Event {
385 timestamp: Some(2000),
386 details: Some(ftest_manager::EventDetails::SuiteStopped(
387 ftest_manager::SuiteStoppedEventDetails {
388 result: Some(ftest_manager::SuiteResult::Finished),
389 ..Default::default()
390 },
391 )),
392 ..Default::default()
393 },
394 ]
395 }
396
397 async fn call_run_tests(params: ParamsForRunTests) -> Outcome {
398 run_test_and_get_outcome(
399 SingleRunConnector::new(params.runner_proxy),
400 params.test_params,
401 RunParams {
402 timeout_behavior: TimeoutBehavior::Continue,
403 timeout_grace_seconds: 0,
404 stop_after_failures: None,
405 accumulate_debug_data: false,
406 log_protocol: None,
407 min_severity_logs: vec![],
408 show_full_moniker: false,
409 },
410 params.run_reporter,
411 futures::future::pending(),
412 )
413 .await
414 }
415
416 #[fuchsia::test]
417 async fn single_run_no_events() {
418 let (runner_proxy, suite_runner_stream) =
419 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
420
421 let reporter = InMemoryReporter::new();
422 let run_reporter = RunReporter::new(reporter.clone());
423 let run_fut = call_run_tests(ParamsForRunTests {
424 runner_proxy,
425 test_params: TestParams {
426 test_url: "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm".to_string(),
427 ..TestParams::default()
428 },
429 run_reporter,
430 });
431 let fake_fut = fake_running_all_suites(
432 suite_runner_stream,
433 hashmap! {
434 "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm" => create_empty_suite_events()
435 },
436 );
437
438 assert_eq!(join(run_fut, fake_fut).await.0, Outcome::Passed,);
439
440 let reports = reporter.get_reports();
441 assert_eq!(2usize, reports.len());
442 assert!(reports[0].report.artifacts.is_empty());
443 assert!(reports[0].report.directories.is_empty());
444 assert!(reports[1].report.artifacts.is_empty());
445 assert!(reports[1].report.directories.is_empty());
446 }
447
448 #[cfg(target_os = "fuchsia")]
449 #[fuchsia::test]
450 async fn single_run_custom_directory() {
451 let (runner_proxy, suite_runner_stream) =
452 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
453
454 let reporter = InMemoryReporter::new();
455 let run_reporter = RunReporter::new(reporter.clone());
456 let run_fut = call_run_tests(ParamsForRunTests {
457 runner_proxy,
458 test_params: TestParams {
459 test_url: "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm".to_string(),
460 ..TestParams::default()
461 },
462 run_reporter,
463 });
464
465 let dir = pseudo_directory! {
466 "test_file.txt" => read_only("Hello, World!"),
467 };
468
469 let directory_proxy = vfs::directory::serve(dir, fio::PERM_READABLE | fio::PERM_WRITABLE);
470
471 let directory_client =
472 fidl::endpoints::ClientEnd::new(directory_proxy.into_channel().unwrap().into());
473
474 let (_pair_1, pair_2) = zx::EventPair::create();
475
476 let events = vec![
477 ftest_manager::Event {
478 timestamp: Some(1000),
479 details: Some(ftest_manager::EventDetails::SuiteStarted(
480 ftest_manager::SuiteStartedEventDetails { ..Default::default() },
481 )),
482 ..Default::default()
483 },
484 ftest_manager::Event {
485 details: Some(ftest_manager::EventDetails::SuiteArtifactGenerated(
486 ftest_manager::SuiteArtifactGeneratedEventDetails {
487 artifact: Some(ftest_manager::Artifact::Custom(
488 ftest_manager::CustomArtifact {
489 directory_and_token: Some(ftest_manager::DirectoryAndToken {
490 directory: directory_client,
491 token: pair_2,
492 }),
493 ..Default::default()
494 },
495 )),
496 ..Default::default()
497 },
498 )),
499 ..Default::default()
500 },
501 ftest_manager::Event {
502 timestamp: Some(2000),
503 details: Some(ftest_manager::EventDetails::SuiteStopped(
504 ftest_manager::SuiteStoppedEventDetails {
505 result: Some(ftest_manager::SuiteResult::Finished),
506 ..Default::default()
507 },
508 )),
509 ..Default::default()
510 },
511 ];
512
513 let fake_fut = fake_running_all_suites(
514 suite_runner_stream,
515 hashmap! {
516 "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm" => events
517 },
518 );
519
520 assert_eq!(join(run_fut, fake_fut).await.0, Outcome::Passed,);
521
522 let reports = reporter.get_reports();
523 assert_eq!(2usize, reports.len());
524 let run = reports.iter().find(|e| e.id == EntityId::Suite).expect("find run report");
525 assert_eq!(1usize, run.report.directories.len());
526 let dir = &run.report.directories[0];
527 let files = dir.1.files.lock();
528 assert_eq!(1usize, files.len());
529 let (name, file) = &files[0];
530 assert_eq!(name.to_string_lossy(), "test_file.txt".to_string());
531 assert_eq!(file.get_contents(), b"Hello, World!");
532 }
533
534 #[fuchsia::test]
535 async fn record_output_after_internal_error() {
536 let (runner_proxy, suite_runner_stream) =
537 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
538
539 let reporter = InMemoryReporter::new();
540 let run_reporter = RunReporter::new(reporter.clone());
541 let run_fut = call_run_tests(ParamsForRunTests {
542 runner_proxy,
543 test_params: TestParams {
544 test_url: "fuchsia-pkg://fuchsia.com/invalid#meta/invalid.cm".to_string(),
545 ..TestParams::default()
546 },
547 run_reporter,
548 });
549
550 let fake_fut = fake_running_all_suites(
551 suite_runner_stream,
552 hashmap! {
553 "fuchsia-pkg://fuchsia.com/invalid#meta/invalid.cm" => vec![
555 ftest_manager::Event {
556 timestamp: Some(1000),
557 details: Some(ftest_manager::EventDetails::SuiteStarted(ftest_manager::SuiteStartedEventDetails {..Default::default()})),
558 ..Default::default()
559 },
560 ftest_manager::Event {
561 timestamp: Some(2000),
562 details: Some(ftest_manager::EventDetails::SuiteStopped(ftest_manager::SuiteStoppedEventDetails {
563 result: Some(ftest_manager::SuiteResult::InternalError),
564 ..Default::default()
565 },
566 )),
567 ..Default::default()
568 },
569 ],
570 },
571 );
572
573 assert_matches!(join(run_fut, fake_fut).await.0, Outcome::Error { .. });
574
575 let reports = reporter.get_reports();
576 assert_eq!(2usize, reports.len());
577 let invalid_suite = reports
578 .iter()
579 .find(|e| e.report.name == "fuchsia-pkg://fuchsia.com/invalid#meta/invalid.cm")
580 .expect("find run report");
581 assert_eq!(invalid_suite.report.outcome, Some(output::ReportedOutcome::Error));
582 assert!(invalid_suite.report.is_finished);
583
584 let run = reports.iter().find(|e| e.id == EntityId::TestRun).expect("find run report");
586 assert_eq!(run.report.outcome, Some(output::ReportedOutcome::Error));
587 assert!(run.report.is_finished);
588 assert!(run.report.started_time.is_some());
589 }
590
591 #[cfg(target_os = "fuchsia")]
592 #[fuchsia::test]
593 async fn single_run_debug_data() {
594 let (runner_proxy, suite_runner_stream) =
595 create_proxy_and_stream::<ftest_manager::SuiteRunnerMarker>();
596
597 let reporter = InMemoryReporter::new();
598 let run_reporter = RunReporter::new(reporter.clone());
599 let run_fut = call_run_tests(ParamsForRunTests {
600 runner_proxy,
601 test_params: TestParams {
602 test_url: "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm".to_string(),
603 ..TestParams::default()
604 },
605 run_reporter,
606 });
607
608 let (debug_client, debug_service) =
609 fidl::endpoints::create_endpoints::<ftest_manager::DebugDataIteratorMarker>();
610 let debug_data_fut = async move {
611 let (client, server) = zx::Socket::create_stream();
612 let mut compressor = zstd::bulk::Compressor::new(0).unwrap();
613 let bytes = compressor.compress(b"Not a real profile").unwrap();
614 let _ = server.write(bytes.as_slice()).unwrap();
615 let mut service = debug_service.into_stream();
616 let mut data = vec![ftest_manager::DebugData {
617 name: Some("test_file.profraw".to_string()),
618 socket: Some(client.into()),
619 ..Default::default()
620 }];
621 drop(server);
622 while let Ok(Some(request)) = service.try_next().await {
623 match request {
624 ftest_manager::DebugDataIteratorRequest::GetNext { .. } => {
625 panic!("Not Implemented");
626 }
627 ftest_manager::DebugDataIteratorRequest::GetNextCompressed {
628 responder,
629 ..
630 } => {
631 let _ = responder.send(std::mem::take(&mut data));
632 }
633 }
634 }
635 };
636
637 let events = vec![
638 ftest_manager::Event {
639 timestamp: Some(1000),
640 details: Some(ftest_manager::EventDetails::SuiteStarted(
641 ftest_manager::SuiteStartedEventDetails { ..Default::default() },
642 )),
643 ..Default::default()
644 },
645 ftest_manager::Event {
646 details: Some(ftest_manager::EventDetails::SuiteArtifactGenerated(
647 ftest_manager::SuiteArtifactGeneratedEventDetails {
648 artifact: Some(ftest_manager::Artifact::DebugData(debug_client)),
649 ..Default::default()
650 },
651 )),
652 ..Default::default()
653 },
654 ftest_manager::Event {
655 timestamp: Some(2000),
656 details: Some(ftest_manager::EventDetails::SuiteStopped(
657 ftest_manager::SuiteStoppedEventDetails {
658 result: Some(ftest_manager::SuiteResult::Finished),
659 ..Default::default()
660 },
661 )),
662 ..Default::default()
663 },
664 ];
665
666 let fake_fut = fake_running_all_suites(
667 suite_runner_stream,
668 hashmap! {
669 "fuchsia-pkg://fuchsia.com/nothing#meta/nothing.cm" => events,
670 },
671 );
672
673 assert_eq!(join3(run_fut, debug_data_fut, fake_fut).await.0, Outcome::Passed);
674
675 let reports = reporter.get_reports();
676 assert_eq!(2usize, reports.len());
677 let run = reports.iter().find(|e| e.id == EntityId::Suite).expect("find run report");
678 assert_eq!(1usize, run.report.directories.len());
679 let dir = &run.report.directories[0];
680 let files = dir.1.files.lock();
681 assert_eq!(1usize, files.len());
682 let (name, file) = &files[0];
683 assert_eq!(name.to_string_lossy(), "test_file.profraw".to_string());
684 assert_eq!(file.get_contents(), b"Not a real profile");
685 }
686}