proptest/test_runner/
config.rs

1//-
2// Copyright 2017, 2018, 2019 The proptest developers
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use crate::std_facade::Box;
11use core::u32;
12
13#[cfg(feature = "std")]
14use std::env;
15#[cfg(feature = "std")]
16use std::ffi::OsString;
17#[cfg(feature = "std")]
18use std::fmt;
19#[cfg(feature = "std")]
20use std::str::FromStr;
21
22use crate::test_runner::result_cache::{noop_result_cache, ResultCache};
23use crate::test_runner::rng::RngAlgorithm;
24use crate::test_runner::FailurePersistence;
25#[cfg(feature = "std")]
26use crate::test_runner::FileFailurePersistence;
27
28#[cfg(feature = "std")]
29const CASES: &str = "PROPTEST_CASES";
30#[cfg(feature = "std")]
31const MAX_LOCAL_REJECTS: &str = "PROPTEST_MAX_LOCAL_REJECTS";
32#[cfg(feature = "std")]
33const MAX_GLOBAL_REJECTS: &str = "PROPTEST_MAX_GLOBAL_REJECTS";
34#[cfg(feature = "std")]
35const MAX_FLAT_MAP_REGENS: &str = "PROPTEST_MAX_FLAT_MAP_REGENS";
36#[cfg(feature = "std")]
37const MAX_SHRINK_TIME: &str = "PROPTEST_MAX_SHRINK_TIME";
38#[cfg(feature = "std")]
39const MAX_SHRINK_ITERS: &str = "PROPTEST_MAX_SHRINK_ITERS";
40#[cfg(feature = "std")]
41const MAX_DEFAULT_SIZE_RANGE: &str = "PROPTEST_MAX_DEFAULT_SIZE_RANGE";
42#[cfg(feature = "fork")]
43const FORK: &str = "PROPTEST_FORK";
44#[cfg(feature = "timeout")]
45const TIMEOUT: &str = "PROPTEST_TIMEOUT";
46#[cfg(feature = "std")]
47const VERBOSE: &str = "PROPTEST_VERBOSE";
48#[cfg(feature = "std")]
49const RNG_ALGORITHM: &str = "PROPTEST_RNG_ALGORITHM";
50#[cfg(feature = "std")]
51const DISABLE_FAILURE_PERSISTENCE: &str =
52    "PROPTEST_DISABLE_FAILURE_PERSISTENCE";
53
54/// Override the config fields from environment variables, if any are set.
55/// Without the `std` feature this function returns config unchanged.
56#[cfg(feature = "std")]
57pub fn contextualize_config(mut result: Config) -> Config {
58    fn parse_or_warn<T: FromStr + fmt::Display>(
59        src: &OsString,
60        dst: &mut T,
61        typ: &str,
62        var: &str,
63    ) {
64        if let Some(src) = src.to_str() {
65            if let Ok(value) = src.parse() {
66                *dst = value;
67            } else {
68                eprintln!(
69                    "proptest: The env-var {}={} can't be parsed as {}, \
70                     using default of {}.",
71                    var, src, typ, *dst
72                );
73            }
74        } else {
75            eprintln!(
76                "proptest: The env-var {} is not valid, using \
77                 default of {}.",
78                var, *dst
79            );
80        }
81    }
82
83    for (var, value) in
84        env::vars_os().filter_map(|(k, v)| k.into_string().ok().map(|k| (k, v)))
85    {
86        match var.as_str() {
87            CASES => parse_or_warn(&value, &mut result.cases, "u32", CASES),
88            MAX_LOCAL_REJECTS => parse_or_warn(
89                &value,
90                &mut result.max_local_rejects,
91                "u32",
92                MAX_LOCAL_REJECTS,
93            ),
94            MAX_GLOBAL_REJECTS => parse_or_warn(
95                &value,
96                &mut result.max_global_rejects,
97                "u32",
98                MAX_GLOBAL_REJECTS,
99            ),
100            MAX_FLAT_MAP_REGENS => parse_or_warn(
101                &value,
102                &mut result.max_flat_map_regens,
103                "u32",
104                MAX_FLAT_MAP_REGENS,
105            ),
106            #[cfg(feature = "fork")]
107            FORK => parse_or_warn(&value, &mut result.fork, "bool", FORK),
108            #[cfg(feature = "timeout")]
109            TIMEOUT => {
110                parse_or_warn(&value, &mut result.timeout, "timeout", TIMEOUT)
111            }
112            MAX_SHRINK_TIME => parse_or_warn(
113                &value,
114                &mut result.max_shrink_time,
115                "u32",
116                MAX_SHRINK_TIME,
117            ),
118            MAX_SHRINK_ITERS => parse_or_warn(
119                &value,
120                &mut result.max_shrink_iters,
121                "u32",
122                MAX_SHRINK_ITERS,
123            ),
124            MAX_DEFAULT_SIZE_RANGE => parse_or_warn(
125                &value,
126                &mut result.max_default_size_range,
127                "usize",
128                MAX_DEFAULT_SIZE_RANGE,
129            ),
130            VERBOSE => {
131                parse_or_warn(&value, &mut result.verbose, "u32", VERBOSE)
132            }
133            RNG_ALGORITHM => parse_or_warn(
134                &value,
135                &mut result.rng_algorithm,
136                "RngAlgorithm",
137                RNG_ALGORITHM,
138            ),
139            DISABLE_FAILURE_PERSISTENCE => result.failure_persistence = None,
140
141            _ => {
142                if var.starts_with("PROPTEST_") {
143                    eprintln!("proptest: Ignoring unknown env-var {}.", var);
144                }
145            }
146        }
147    }
148
149    result
150}
151
152/// Without the `std` feature this function returns config unchanged.
153#[cfg(not(feature = "std"))]
154pub fn contextualize_config(result: Config) -> Config {
155    result
156}
157
158fn default_default_config() -> Config {
159    Config {
160        cases: 256,
161        max_local_rejects: 65_536,
162        max_global_rejects: 1024,
163        max_flat_map_regens: 1_000_000,
164        failure_persistence: None,
165        source_file: None,
166        test_name: None,
167        #[cfg(feature = "fork")]
168        fork: false,
169        #[cfg(feature = "timeout")]
170        timeout: 0,
171        #[cfg(feature = "std")]
172        max_shrink_time: 0,
173        max_shrink_iters: u32::MAX,
174        max_default_size_range: 100,
175        result_cache: noop_result_cache,
176        #[cfg(feature = "std")]
177        verbose: 0,
178        rng_algorithm: RngAlgorithm::default(),
179        _non_exhaustive: (),
180    }
181}
182
183// The default config, computed by combining environment variables and
184// defaults.
185#[cfg(feature = "std")]
186lazy_static! {
187    static ref DEFAULT_CONFIG: Config = {
188        let mut default_config = default_default_config();
189        default_config.failure_persistence = Some(Box::new(FileFailurePersistence::default()));
190        contextualize_config(default_config)
191    };
192}
193
194/// Configuration for how a proptest test should be run.
195#[derive(Clone, Debug, PartialEq)]
196pub struct Config {
197    /// The number of successful test cases that must execute for the test as a
198    /// whole to pass.
199    ///
200    /// This does not include implicitly-replayed persisted failing cases.
201    ///
202    /// The default is 256, which can be overridden by setting the
203    /// `PROPTEST_CASES` environment variable. (The variable is only considered
204    /// when the `std` feature is enabled, which it is by default.)
205    pub cases: u32,
206
207    /// The maximum number of individual inputs that may be rejected before the
208    /// test as a whole aborts.
209    ///
210    /// The default is 65536, which can be overridden by setting the
211    /// `PROPTEST_MAX_LOCAL_REJECTS` environment variable. (The variable is only
212    /// considered when the `std` feature is enabled, which it is by default.)
213    pub max_local_rejects: u32,
214
215    /// The maximum number of combined inputs that may be rejected before the
216    /// test as a whole aborts.
217    ///
218    /// The default is 1024, which can be overridden by setting the
219    /// `PROPTEST_MAX_GLOBAL_REJECTS` environment variable. (The variable is
220    /// only considered when the `std` feature is enabled, which it is by
221    /// default.)
222    pub max_global_rejects: u32,
223
224    /// The maximum number of times all `Flatten` combinators will attempt to
225    /// regenerate values. This puts a limit on the worst-case exponential
226    /// explosion that can happen with nested `Flatten`s.
227    ///
228    /// The default is 1_000_000, which can be overridden by setting the
229    /// `PROPTEST_MAX_FLAT_MAP_REGENS` environment variable. (The variable is
230    /// only considered when the `std` feature is enabled, which it is by
231    /// default.)
232    pub max_flat_map_regens: u32,
233
234    /// Indicates whether and how to persist failed test results.
235    ///
236    /// When compiling with "std" feature (i.e. the standard library is available), the default
237    /// is `Some(Box::new(FileFailurePersistence::SourceParallel("proptest-regressions")))`.
238    ///
239    /// Without the standard library, the default is `None`, and no persistence occurs.
240    ///
241    /// See the docs of [`FileFailurePersistence`](enum.FileFailurePersistence.html)
242    /// and [`MapFailurePersistence`](struct.MapFailurePersistence.html) for more information.
243    ///
244    /// You can disable failure persistence with the `PROPTEST_DISABLE_FAILURE_PERSISTENCE`
245    /// environment variable but its not currently possible to set the persistence file
246    /// with an environment variable. (The variable is
247    /// only considered when the `std` feature is enabled, which it is by
248    /// default.)
249    pub failure_persistence: Option<Box<dyn FailurePersistence>>,
250
251    /// File location of the current test, relevant for persistence
252    /// and debugging.
253    ///
254    /// Note the use of `&str` rather than `Path` to be compatible with
255    /// `#![no_std]` use cases where `Path` is unavailable.
256    ///
257    /// See the docs of [`FileFailurePersistence`](enum.FileFailurePersistence.html)
258    /// for more information on how it may be used for persistence.
259    pub source_file: Option<&'static str>,
260
261    /// The fully-qualified name of the test being run, as would be passed to
262    /// the test executable to run just that test.
263    ///
264    /// This must be set if `fork` is `true`. Otherwise, it is unused. It is
265    /// automatically set by `proptest!`.
266    ///
267    /// This must include the crate name at the beginning, as produced by
268    /// `module_path!()`.
269    pub test_name: Option<&'static str>,
270
271    /// If true, tests are run in a subprocess.
272    ///
273    /// Forking allows proptest to work with tests which may fail by aborting
274    /// the process, causing a segmentation fault, etc, but can be a lot slower
275    /// in certain environments or when running a very large number of tests.
276    ///
277    /// For forking to work correctly, both the `Strategy` and the content of
278    /// the test case itself must be deterministic.
279    ///
280    /// This requires the "fork" feature, enabled by default.
281    ///
282    /// The default is `false`, which can be overridden by setting the
283    /// `PROPTEST_FORK` environment variable. (The variable is
284    /// only considered when the `std` feature is enabled, which it is by
285    /// default.)
286    #[cfg(feature = "fork")]
287    #[cfg_attr(docsrs, doc(cfg(feature = "fork")))]
288    pub fork: bool,
289
290    /// If non-zero, tests are run in a subprocess and each generated case
291    /// fails if it takes longer than this number of milliseconds.
292    ///
293    /// This implicitly enables forking, even if the `fork` field is `false`.
294    ///
295    /// The type here is plain `u32` (rather than
296    /// `Option<std::time::Duration>`) for the sake of ergonomics.
297    ///
298    /// This requires the "timeout" feature, enabled by default.
299    ///
300    /// Setting a timeout to less than the time it takes the process to start
301    /// up and initialise the first test case will cause the whole test to be
302    /// aborted.
303    ///
304    /// The default is `0` (i.e., no timeout), which can be overridden by
305    /// setting the `PROPTEST_TIMEOUT` environment variable. (The variable is
306    /// only considered when the `std` feature is enabled, which it is by
307    /// default.)
308    #[cfg(feature = "timeout")]
309    #[cfg_attr(docsrs, doc(cfg(feature = "timeout")))]
310    pub timeout: u32,
311
312    /// If non-zero, give up the shrinking process after this many milliseconds
313    /// have elapsed since the start of the shrinking process.
314    ///
315    /// This will not cause currently running test cases to be interrupted.
316    ///
317    /// This configuration is only available when the `std` feature is enabled
318    /// (which it is by default).
319    ///
320    /// The default is `0` (i.e., no limit), which can be overridden by setting
321    /// the `PROPTEST_MAX_SHRINK_TIME` environment variable. (The variable is
322    /// only considered when the `std` feature is enabled, which it is by
323    /// default.)
324    #[cfg(feature = "std")]
325    #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
326    pub max_shrink_time: u32,
327
328    /// Give up on shrinking if more than this number of iterations of the test
329    /// code are run.
330    ///
331    /// Setting this to `std::u32::MAX` causes the actual limit to be four
332    /// times the number of test cases.
333    ///
334    /// Setting this value to `0` disables shrinking altogether.
335    ///
336    /// Note that the type of this field will change in a future version of
337    /// proptest to better accommodate its special values.
338    ///
339    /// The default is `std::u32::MAX`, which can be overridden by setting the
340    /// `PROPTEST_MAX_SHRINK_ITERS` environment variable. (The variable is only
341    /// considered when the `std` feature is enabled, which it is by default.)
342    pub max_shrink_iters: u32,
343
344    /// The default maximum size to `proptest::collection::SizeRange`. The default
345    /// strategy for collections (like `Vec`) use collections in the range of
346    /// `0..max_default_size_range`.
347    ///
348    /// The default is `100` which can be overridden by setting the
349    /// `PROPTEST_MAX_DEFAULT_SIZE_RANGE` environment variable. (The variable
350    /// is only considered when the `std` feature is enabled, which it is by
351    /// default.)
352    pub max_default_size_range: usize,
353
354    /// A function to create new result caches.
355    ///
356    /// The default is to do no caching. The easiest way to enable caching is
357    /// to set this field to `basic_result_cache` (though that is currently
358    /// only available with the `std` feature).
359    ///
360    /// This is useful for strategies which have a tendency to produce
361    /// duplicate values, or for tests where shrinking can take a very long
362    /// time due to exploring the same output multiple times.
363    ///
364    /// When caching is enabled, generated values themselves are not stored, so
365    /// this does not pose a risk of memory exhaustion for large test inputs
366    /// unless using extraordinarily large test case counts.
367    ///
368    /// Caching incurs its own overhead, and may very well make your test run
369    /// more slowly.
370    pub result_cache: fn() -> Box<dyn ResultCache>,
371
372    /// Set to non-zero values to cause proptest to emit human-targeted
373    /// messages to stderr as it runs.
374    ///
375    /// Greater values cause greater amounts of logs to be emitted. The exact
376    /// meaning of certain levels other than 0 is subject to change.
377    ///
378    /// - 0: No extra output.
379    /// - 1: Log test failure messages. In state machine tests, this level is
380    ///   used to print transitions.
381    /// - 2: Trace low-level details.
382    ///
383    /// This is only available with the `std` feature (enabled by default)
384    /// since on nostd proptest has no way to produce output.
385    ///
386    /// The default is `0`, which can be overridden by setting the
387    /// `PROPTEST_VERBOSE` environment variable. (The variable is only considered
388    /// when the `std` feature is enabled, which it is by default.)
389    #[cfg(feature = "std")]
390    #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
391    pub verbose: u32,
392
393    /// The RNG algorithm to use when not using a user-provided RNG.
394    ///
395    /// The default is `RngAlgorithm::default()`, which can be overridden by
396    /// setting the `PROPTEST_RNG_ALGORITHM` environment variable to one of the following:
397    ///
398    /// - `xs` — `RngAlgorithm::XorShift`
399    /// - `cc` — `RngAlgorithm::ChaCha`
400    ///
401    /// (The variable is only considered when the `std` feature is enabled,
402    /// which it is by default.)
403    pub rng_algorithm: RngAlgorithm,
404
405    // Needs to be public so FRU syntax can be used.
406    #[doc(hidden)]
407    pub _non_exhaustive: (),
408}
409
410impl Config {
411    /// Constructs a `Config` only differing from the `default()` in the
412    /// number of test cases required to pass the test successfully.
413    ///
414    /// This is simply a more concise alternative to using field-record update
415    /// syntax:
416    ///
417    /// ```
418    /// # use proptest::test_runner::Config;
419    /// assert_eq!(
420    ///     Config::with_cases(42),
421    ///     Config { cases: 42, .. Config::default() }
422    /// );
423    /// ```
424    pub fn with_cases(cases: u32) -> Self {
425        Self {
426            cases,
427            ..Config::default()
428        }
429    }
430
431    /// Constructs a `Config` only differing from the `default()` in the
432    /// source_file of the present test.
433    ///
434    /// This is simply a more concise alternative to using field-record update
435    /// syntax:
436    ///
437    /// ```
438    /// # use proptest::test_runner::Config;
439    /// assert_eq!(
440    ///     Config::with_source_file("computer/question"),
441    ///     Config { source_file: Some("computer/question"), .. Config::default() }
442    /// );
443    /// ```
444    pub fn with_source_file(source_file: &'static str) -> Self {
445        Self {
446            source_file: Some(source_file),
447            ..Config::default()
448        }
449    }
450
451    /// Constructs a `Config` only differing from the provided Config instance, `self`,
452    /// in the source_file of the present test.
453    ///
454    /// This is simply a more concise alternative to using field-record update
455    /// syntax:
456    ///
457    /// ```
458    /// # use proptest::test_runner::Config;
459    /// let a = Config::with_source_file("computer/question");
460    /// let b = a.clone_with_source_file("answer/42");
461    /// assert_eq!(
462    ///     a,
463    ///     Config { source_file: Some("computer/question"), .. Config::default() }
464    /// );
465    /// assert_eq!(
466    ///     b,
467    ///     Config { source_file: Some("answer/42"), .. Config::default() }
468    /// );
469    /// ```
470    pub fn clone_with_source_file(&self, source_file: &'static str) -> Self {
471        let mut result = self.clone();
472        result.source_file = Some(source_file);
473        result
474    }
475
476    /// Return whether this configuration implies forking.
477    ///
478    /// This method exists even if the "fork" feature is disabled, in which
479    /// case it simply returns false.
480    pub fn fork(&self) -> bool {
481        self._fork() || self.timeout() > 0
482    }
483
484    #[cfg(feature = "fork")]
485    fn _fork(&self) -> bool {
486        self.fork
487    }
488
489    #[cfg(not(feature = "fork"))]
490    fn _fork(&self) -> bool {
491        false
492    }
493
494    /// Returns the configured timeout.
495    ///
496    /// This method exists even if the "timeout" feature is disabled, in which
497    /// case it simply returns 0.
498    #[cfg(feature = "timeout")]
499    pub fn timeout(&self) -> u32 {
500        self.timeout
501    }
502
503    /// Returns the configured timeout.
504    ///
505    /// This method exists even if the "timeout" feature is disabled, in which
506    /// case it simply returns 0.
507    #[cfg(not(feature = "timeout"))]
508    pub fn timeout(&self) -> u32 {
509        0
510    }
511
512    /// Returns the configured limit on shrinking iterations.
513    ///
514    /// This takes into account the special "automatic" behaviour.
515    pub fn max_shrink_iters(&self) -> u32 {
516        if u32::MAX == self.max_shrink_iters {
517            self.cases.saturating_mul(4)
518        } else {
519            self.max_shrink_iters
520        }
521    }
522
523    // Used by macros to force the config to be owned without depending on
524    // certain traits being `use`d.
525    #[allow(missing_docs)]
526    #[doc(hidden)]
527    pub fn __sugar_to_owned(&self) -> Self {
528        self.clone()
529    }
530}
531
532#[cfg(feature = "std")]
533impl Default for Config {
534    fn default() -> Self {
535        DEFAULT_CONFIG.clone()
536    }
537}
538
539#[cfg(not(feature = "std"))]
540impl Default for Config {
541    fn default() -> Self {
542        default_default_config()
543    }
544}