run_test_suite_lib/
outcome.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use fidl_fuchsia_test_manager as ftest_manager;
6use std::fmt;
7use std::sync::Arc;
8use thiserror::Error;
9
10#[derive(Debug, Clone)]
11pub enum Outcome {
12    Passed,
13    Failed,
14    Inconclusive,
15    Timedout,
16    /// Suite was stopped prematurely due to cancellation by the user.
17    Cancelled,
18    /// Suite did not report completion.
19    // TODO(https://fxbug.dev/42171445) - this outcome indicates an internal error as test manager isn't
20    // sending expected events. We should return an error instead.
21    DidNotFinish,
22    Error {
23        origin: Arc<RunTestSuiteError>,
24    },
25}
26
27impl Outcome {
28    pub(crate) fn error<E: Into<RunTestSuiteError>>(e: E) -> Self {
29        Self::Error { origin: Arc::new(e.into()) }
30    }
31}
32
33impl PartialEq for Outcome {
34    fn eq(&self, other: &Self) -> bool {
35        match (self, other) {
36            (Self::Passed, Self::Passed)
37            | (Self::Failed, Self::Failed)
38            | (Self::Inconclusive, Self::Inconclusive)
39            | (Self::Timedout, Self::Timedout)
40            | (Self::Cancelled, Self::Cancelled)
41            | (Self::DidNotFinish, Self::DidNotFinish) => true,
42            (Self::Error { origin }, Self::Error { origin: other_origin }) => {
43                format!("{}", origin.as_ref()) == format!("{}", other_origin.as_ref())
44            }
45            (_, _) => false,
46        }
47    }
48}
49
50impl fmt::Display for Outcome {
51    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52        match self {
53            Outcome::Passed => write!(f, "PASSED"),
54            Outcome::Failed => write!(f, "FAILED"),
55            Outcome::Inconclusive => write!(f, "INCONCLUSIVE"),
56            Outcome::Timedout => write!(f, "TIMED OUT"),
57            Outcome::Cancelled => write!(f, "CANCELLED"),
58            Outcome::DidNotFinish => write!(f, "DID_NOT_FINISH"),
59            Outcome::Error { .. } => write!(f, "ERROR"),
60        }
61    }
62}
63
64#[derive(Error, Debug)]
65/// An enum of the different errors that may be encountered while running
66/// a test.
67pub enum RunTestSuiteError {
68    #[error("fidl error: {0:?}")]
69    Fidl(#[from] fidl::Error),
70    #[error("error launching test suite: {}", convert_launch_error_to_str(.0))]
71    Launch(ftest_manager::LaunchError),
72    #[error("error reporting test results: {0:?}")]
73    Io(#[from] std::io::Error),
74    #[error("unexpected event: {0:?}")]
75    UnexpectedEvent(#[from] UnexpectedEventError),
76    #[error("Error connecting to RunBuilder protocol: {0:?}")]
77    Connection(#[from] ConnectionError),
78}
79
80/// An error returned when test manager reports an unexpected event.
81/// This could occur if test manager violates guarantees about event
82/// ordering.
83#[derive(Error, Debug)]
84pub enum UnexpectedEventError {
85    #[error(
86        "received a 'started' event for case with id {identifier:?} but no 'case_found' event"
87    )]
88    CaseStartedButNotFound { identifier: u32 },
89    #[error(
90        "invalid case event to '{next_state:?}' received while in state '{last_state:?}' for case {test_case_name:?} with id {identifier:?}"
91    )]
92    InvalidCaseEvent {
93        last_state: Lifecycle,
94        next_state: Lifecycle,
95        test_case_name: String,
96        identifier: u32,
97    },
98    #[error(
99        "received an 'artifact' event for case with id {identifier:?} but no 'case_found' event"
100    )]
101    CaseArtifactButNotFound { identifier: u32 },
102    #[error(
103        "received an 'artifact' event for case with id {identifier:?} but the case is already finished"
104    )]
105    CaseArtifactButFinished { identifier: u32 },
106    #[error(
107        "received a '{next_state:?}' event for case with id {identifier:?} but no 'case_found' event"
108    )]
109    CaseEventButNotFound { next_state: Lifecycle, identifier: u32 },
110    #[error("received a 'stopped' event for case with id {identifier:?} but no 'started' event")]
111    UnrecognizedCaseStatus { status: ftest_manager::CaseStatus, identifier: u32 },
112    #[error("server closed channel without reporting finish for cases: {cases:?}")]
113    CasesDidNotFinish { cases: Vec<String> },
114    #[error("invalid suite event to '{next_state:?}' received while in state '{last_state:?}'")]
115    InvalidSuiteEvent { last_state: Lifecycle, next_state: Lifecycle },
116    #[error("received an unhandled suite status: {status:?}")]
117    UnrecognizedSuiteStatus { status: ftest_manager::SuiteStatus },
118    #[error("server closed channel without reporting a result for the suite")]
119    SuiteDidNotReportStop,
120    #[error("received an InternalError suite status")]
121    InternalErrorSuiteStatus,
122    #[error("missing required field {field} in {containing_struct}")]
123    MissingRequiredField { containing_struct: &'static str, field: &'static str },
124}
125
126#[derive(Debug, Error)]
127#[error(transparent)]
128pub struct ConnectionError(pub anyhow::Error);
129
130/// Lifecycle of a test suite or test case.
131/// This is internal implementation for ::crate::run, but is located here so it can be reported
132/// for debugging via RunTestSuiteError.
133#[derive(Clone, Copy, Debug)]
134pub enum Lifecycle {
135    Found,
136    Started,
137    Stopped,
138    Finished,
139}
140
141impl RunTestSuiteError {
142    /// Returns true iff the error variant indicates an internal error in
143    /// Test Manager or ffx.
144    pub fn is_internal_error(&self) -> bool {
145        match self {
146            Self::Fidl(_) => true,
147            Self::Launch(ftest_manager::LaunchError::InternalError) => true,
148            Self::Launch(_) => false,
149            Self::Io(_) => true,
150            Self::UnexpectedEvent(_) => true,
151            Self::Connection(_) => true,
152        }
153    }
154}
155
156impl From<ftest_manager::LaunchError> for RunTestSuiteError {
157    fn from(launch: ftest_manager::LaunchError) -> Self {
158        Self::Launch(launch)
159    }
160}
161
162fn convert_launch_error_to_str(e: &ftest_manager::LaunchError) -> &'static str {
163    match e {
164        ftest_manager::LaunchError::CaseEnumeration => "Cannot enumerate test. This may mean `fuchsia.test.Suite` was not configured correctly. Refer to: \
165        https://fuchsia.dev/go/components/test-errors",
166        ftest_manager::LaunchError::ResourceUnavailable => "Resource unavailable",
167        ftest_manager::LaunchError::InstanceCannotResolve => "Cannot resolve test.",
168        ftest_manager::LaunchError::InvalidArgs => {
169            "Invalid args passed to builder while adding suite. Please file bug"
170        }
171        ftest_manager::LaunchError::FailedToConnectToTestSuite => {
172            "Cannot communicate with the tests. This may mean `fuchsia.test.Suite` was not \
173            configured correctly. Refer to: \
174            https://fuchsia.dev/go/components/test-errors"
175        }
176        ftest_manager::LaunchError::InternalError => "Internal error, please file bug",
177        ftest_manager::LaunchError::NoMatchingCases =>
178            // TODO(satsukiu): Ideally, we would expose these error enums up through the library
179            // and define the error messages in the main files for each tool. This would allow each
180            // tool (ffx/fx/run-test-suite) to give the correct flags.
181            "No test cases matched the specified filters.\n\
182            If you specified a test filter, verify the available test cases with \
183            'ffx test list-cases <test suite url>'.\n\
184            If the list of available tests contains only a single test case called either \
185            'legacy_test' or 'main', the suite likely uses either the legacy_test_runner or \
186            elf_test_runner. In these cases, --test-filter will not work. Instead, \
187            you can pass test arguments directly to the test instead. Refer to: \
188            https://fuchsia.dev/go/components/test-runners",
189        ftest_manager::LaunchError::InvalidManifest => "The test manifest is invalid or has invalid facets/arguments. Please check logs for detailed error.",
190        ftest_manager::LaunchErrorUnknown!() => "Unrecognized launch error",
191    }
192}
run_test_suite_lib/outcome.rs

run_test_suite_lib/
outcome.rs
