Add --max-success for running more or less prop runs on demand.

CHANGELOG.md

@@ -17,6 +17,7 @@
 - **aiken**: support outputting mainnet addresses for validators. @rvcas
 - **aiken-lang**: added serde to CheckedModule to encode modules as cbor. @rvcas
 - **aiken-lang**: Strings can contain a nul byte using the escape sequence `\0`. @KtorZ
+- **aiken**: The `check` command now accept an extra (optional) option `--max-success` to control the number of property-test iterations to perform. @KtorZ
 
 ### Fixed
 

crates/aiken-lsp/src/server/lsp_project.rs

@@ -1,5 +1,8 @@
 use aiken_lang::{ast::Tracing, line_numbers::LineNumbers};
-use aiken_project::{config::Config, error::Error as ProjectError, module::CheckedModule, Project};
+use aiken_project::{
+    config::Config, error::Error as ProjectError, module::CheckedModule,
+    test_framework::PropertyTest, Project,
+};
 use std::{collections::HashMap, path::PathBuf};
 
 #[derive(Debug)]
@@ -29,9 +32,15 @@ impl LspProject {
     pub fn compile(&mut self) -> Result<(), Vec<ProjectError>> {
         let checkpoint = self.project.checkpoint();
 
-        let result =
-            self.project
-                .check(true, None, false, false, u32::default(), Tracing::silent());
+        let result = self.project.check(
+            true,
+            None,
+            false,
+            false,
+            u32::default(),
+            PropertyTest::DEFAULT_MAX_SUCCESS,
+            Tracing::silent(),
+        );
 
         self.project.restore(checkpoint);
 

crates/aiken-project/src/lib.rs

@@ -223,6 +223,7 @@ where
         Ok(())
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn check(
         &mut self,
         skip_tests: bool,
@@ -230,6 +231,7 @@ where
         verbose: bool,
         exact_match: bool,
         seed: u32,
+        property_max_success: usize,
         tracing: Tracing,
     ) -> Result<(), Vec<Error>> {
         let options = Options {
@@ -242,6 +244,7 @@ where
                     verbose,
                     exact_match,
                     seed,
+                    property_max_success,
                 }
             },
         };
@@ -328,6 +331,7 @@ where
                 verbose,
                 exact_match,
                 seed,
+                property_max_success,
             } => {
                 let tests =
                     self.collect_tests(verbose, match_tests, exact_match, options.tracing)?;
@@ -336,7 +340,7 @@ where
                     self.event_listener.handle_event(Event::RunningTests);
                 }
 
-                let tests = self.run_tests(tests, seed);
+                let tests = self.run_tests(tests, seed, property_max_success);
 
                 self.checks_count = if tests.is_empty() {
                     None
@@ -848,7 +852,12 @@ where
         Ok(tests)
     }
 
-    fn run_tests(&self, tests: Vec<Test>, seed: u32) -> Vec<TestResult<UntypedExpr, UntypedExpr>> {
+    fn run_tests(
+        &self,
+        tests: Vec<Test>,
+        seed: u32,
+        property_max_success: usize,
+    ) -> Vec<TestResult<UntypedExpr, UntypedExpr>> {
         use rayon::prelude::*;
 
         let data_types = utils::indexmap::as_ref_values(&self.data_types);
@@ -857,7 +866,7 @@ where
             .into_par_iter()
             .map(|test| match test {
                 Test::UnitTest(unit_test) => unit_test.run(),
-                Test::PropertyTest(property_test) => property_test.run(seed),
+                Test::PropertyTest(property_test) => property_test.run(seed, property_max_success),
             })
             .collect::<Vec<TestResult<(Constant, Rc<Type>), PlutusData>>>()
             .into_iter()

crates/aiken-project/src/options.rs

@@ -11,6 +11,7 @@ pub enum CodeGenMode {
         verbose: bool,
         exact_match: bool,
         seed: u32,
+        property_max_success: usize,
     },
     Build(bool),
     NoOp,

crates/aiken-project/src/test_framework.rs

@@ -214,13 +214,11 @@ pub struct Fuzzer<T> {
 }
 
 impl PropertyTest {
-    const MAX_TEST_RUN: usize = 100;
+    pub const DEFAULT_MAX_SUCCESS: usize = 100;
 
-    /// Run a property test from a given seed. The property is run at most MAX_TEST_RUN times. It
+    /// Run a property test from a given seed. The property is run at most DEFAULT_MAX_SUCCESS times. It
     /// may stops earlier on failure; in which case a 'counterexample' is returned.
-    pub fn run<U>(self, seed: u32) -> TestResult<U, PlutusData> {
-        let n = PropertyTest::MAX_TEST_RUN;
-
+    pub fn run<U>(self, seed: u32, n: usize) -> TestResult<U, PlutusData> {
         let mut labels = BTreeMap::new();
 
         let (counterexample, iterations) =
@@ -1378,7 +1376,7 @@ mod test {
         fn expect_failure(&self) -> Counterexample {
             let mut labels = BTreeMap::new();
             match self.run_n_times(
-                PropertyTest::MAX_TEST_RUN,
+                PropertyTest::DEFAULT_MAX_SUCCESS,
                 Prng::from_seed(42),
                 None,
                 &mut labels,
@@ -1397,7 +1395,9 @@ mod test {
             }
         "#});
 
-        assert!(prop.run::<()>(42).is_success());
+        assert!(prop
+            .run::<()>(42, PropertyTest::DEFAULT_MAX_SUCCESS)
+            .is_success());
     }
 
     #[test]
@@ -1419,7 +1419,7 @@ mod test {
             }
         "#});
 
-        match prop.run::<()>(42) {
+        match prop.run::<()>(42, PropertyTest::DEFAULT_MAX_SUCCESS) {
             TestResult::UnitTestResult(..) => unreachable!("property returned unit-test result ?!"),
             TestResult::PropertyTestResult(result) => {
                 assert!(

crates/aiken/src/cmd/check.rs

@@ -1,6 +1,9 @@
 use super::build::{filter_traces_parser, trace_level_parser};
 use aiken_lang::ast::{TraceLevel, Tracing};
-use aiken_project::watch::{self, watch_project, with_project};
+use aiken_project::{
+    test_framework::PropertyTest,
+    watch::{self, watch_project, with_project},
+};
 use rand::prelude::*;
 use std::{path::PathBuf, process};
 
@@ -30,6 +33,10 @@ pub struct Args {
     #[clap(long)]
     seed: Option<u32>,
 
+    /// Maximum number of successful test run for considering a property-based test valid.
+    #[clap(long, default_value_t = PropertyTest::DEFAULT_MAX_SUCCESS)]
+    max_success: usize,
+
     /// Only run tests if they match any of these strings.
     /// You can match a module with `-m aiken/list` or `-m list`.
     /// You can match a test with `-m "aiken/list.{map}"` or `-m "aiken/option.{flatten_1}"`
@@ -72,6 +79,7 @@ pub fn exec(
         filter_traces,
         trace_level,
         seed,
+        max_success,
     }: Args,
 ) -> miette::Result<()> {
     let mut rng = rand::thread_rng();
@@ -86,6 +94,7 @@ pub fn exec(
                 debug,
                 exact_match,
                 seed,
+                max_success,
                 match filter_traces {
                     Some(filter_traces) => filter_traces(trace_level),
                     None => Tracing::All(trace_level),
@@ -100,6 +109,7 @@ pub fn exec(
                 debug,
                 exact_match,
                 seed,
+                max_success,
                 match filter_traces {
                     Some(filter_traces) => filter_traces(trace_level),
                     None => Tracing::All(trace_level),