Introduce 'fail once' and alter behavior of 'fail' keyword for properties.

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v1.0.29-alpha - UNRELEASED
+
+### Changed
+
+- **aiken-lang**: the keyword `fail` on property-based test semantic has changed and now consider a test to succeed only if **every** execution of the test failed (instead of just one). The previous behavior can be recovered by adding the keyword `once` after `fail`. @KtorZ
+
 ## v1.0.28-alpha - 2024-05-23
 
 ### Added

crates/aiken-lang/src/ast.rs

@@ -223,6 +223,13 @@ pub type UntypedFunction = Function<(), UntypedExpr, UntypedArg>;
 pub type TypedTest = Function<Rc<Type>, TypedExpr, TypedArgVia>;
 pub type UntypedTest = Function<(), UntypedExpr, UntypedArgVia>;
 
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub enum OnTestFailure {
+    FailImmediately,
+    SucceedImmediately,
+    SucceedEventually,
+}
+
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub struct Function<T, Expr, Arg> {
     pub arguments: Vec<Arg>,
@@ -234,7 +241,7 @@ pub struct Function<T, Expr, Arg> {
     pub return_annotation: Option<Annotation>,
     pub return_type: T,
     pub end_position: usize,
-    pub can_error: bool,
+    pub on_test_failure: OnTestFailure,
 }
 
 impl TypedFunction {
@@ -279,7 +286,7 @@ impl From<UntypedTest> for UntypedFunction {
             return_annotation: f.return_annotation,
             return_type: f.return_type,
             body: f.body,
-            can_error: f.can_error,
+            on_test_failure: f.on_test_failure,
             end_position: f.end_position,
         }
     }
@@ -296,7 +303,7 @@ impl From<TypedTest> for TypedFunction {
             return_annotation: f.return_annotation,
             return_type: f.return_type,
             body: f.body,
-            can_error: f.can_error,
+            on_test_failure: f.on_test_failure,
             end_position: f.end_position,
         }
     }

crates/aiken-lang/src/builtins.rs

@@ -1,7 +1,7 @@
 use crate::{
     ast::{
         Annotation, Arg, ArgName, CallArg, DataTypeKey, Function, FunctionAccessKey, ModuleKind,
-        Span, TypedDataType, TypedFunction, UnOp,
+        OnTestFailure, Span, TypedDataType, TypedFunction, UnOp,
     },
     expr::TypedExpr,
     tipo::{
@@ -944,7 +944,7 @@ pub fn prelude_functions(id_gen: &IdGenerator) -> IndexMap<FunctionAccessKey, Ty
                 annotation: None,
                 tipo: bool(),
             }],
-            can_error: false,
+            on_test_failure: OnTestFailure::FailImmediately,
             doc: Some(
                 indoc::indoc! {
                     r#"
@@ -1001,7 +1001,7 @@ pub fn prelude_functions(id_gen: &IdGenerator) -> IndexMap<FunctionAccessKey, Ty
                 doc: None,
                 tipo: a_var.clone(),
             }],
-            can_error: false,
+            on_test_failure: OnTestFailure::FailImmediately,
             body: TypedExpr::Var {
                 location: Span::empty(),
                 constructor: ValueConstructor {
@@ -1043,7 +1043,7 @@ pub fn prelude_functions(id_gen: &IdGenerator) -> IndexMap<FunctionAccessKey, Ty
             function_name: "always".to_string(),
         },
         Function {
-            can_error: false,
+            on_test_failure: OnTestFailure::FailImmediately,
             arguments: vec![
                 Arg {
                     arg_name: ArgName::Named {
@@ -1121,7 +1121,7 @@ pub fn prelude_functions(id_gen: &IdGenerator) -> IndexMap<FunctionAccessKey, Ty
             function_name: "flip".to_string(),
         },
         Function {
-            can_error: false,
+            on_test_failure: OnTestFailure::FailImmediately,
             arguments: vec![Arg {
                 arg_name: ArgName::Named {
                     name: "f".to_string(),

crates/aiken-lang/src/format.rs

@@ -2,11 +2,11 @@ use crate::{
     ast::{
         Annotation, Arg, ArgName, ArgVia, AssignmentKind, AssignmentPattern, BinOp,
         ByteArrayFormatPreference, CallArg, ClauseGuard, Constant, CurveType, DataType, Definition,
-        Function, IfBranch, LogicalOpChainKind, ModuleConstant, Pattern, RecordConstructor,
-        RecordConstructorArg, RecordUpdateSpread, Span, TraceKind, TypeAlias, TypedArg, UnOp,
-        UnqualifiedImport, UntypedArg, UntypedArgVia, UntypedAssignmentKind, UntypedClause,
-        UntypedClauseGuard, UntypedDefinition, UntypedFunction, UntypedModule, UntypedPattern,
-        UntypedRecordUpdateArg, Use, Validator, CAPTURE_VARIABLE,
+        Function, IfBranch, LogicalOpChainKind, ModuleConstant, OnTestFailure, Pattern,
+        RecordConstructor, RecordConstructorArg, RecordUpdateSpread, Span, TraceKind, TypeAlias,
+        TypedArg, UnOp, UnqualifiedImport, UntypedArg, UntypedArgVia, UntypedAssignmentKind,
+        UntypedClause, UntypedClauseGuard, UntypedDefinition, UntypedFunction, UntypedModule,
+        UntypedPattern, UntypedRecordUpdateArg, Use, Validator, CAPTURE_VARIABLE,
     },
     docvec,
     expr::{FnStyle, UntypedExpr, DEFAULT_ERROR_STR, DEFAULT_TODO_STR},
@@ -247,9 +247,9 @@ impl<'comments> Formatter<'comments> {
                 arguments: args,
                 body,
                 end_position,
-                can_error,
+                on_test_failure,
                 ..
-            }) => self.definition_test(name, args, body, *end_position, *can_error),
+            }) => self.definition_test(name, args, body, *end_position, on_test_failure),
 
             Definition::TypeAlias(TypeAlias {
                 alias,
@@ -547,14 +547,18 @@ impl<'comments> Formatter<'comments> {
         args: &'a [UntypedArgVia],
         body: &'a UntypedExpr,
         end_location: usize,
-        can_error: bool,
+        on_test_failure: &'a OnTestFailure,
     ) -> Document<'a> {
         // Fn name and args
         let head = "test "
             .to_doc()
             .append(name)
             .append(wrap_args(args.iter().map(|e| (self.fn_arg_via(e), false))))
-            .append(if can_error { " fail" } else { "" })
+            .append(match on_test_failure {
+                OnTestFailure::FailImmediately => "",
+                OnTestFailure::SucceedEventually => " fail",
+                OnTestFailure::SucceedImmediately => " fail once",
+            })
             .group();
 
         // Format body

crates/aiken-lang/src/parser/definition/function.rs

@@ -1,10 +1,9 @@
-use chumsky::prelude::*;
-
 use crate::{
     ast,
     expr::UntypedExpr,
     parser::{annotation, error::ParseError, expr, token::Token, utils},
 };
+use chumsky::prelude::*;
 
 pub fn parser() -> impl Parser<Token, ast::UntypedDefinition, Error = ParseError> {
     utils::optional_flag(Token::Pub)
@@ -41,7 +40,7 @@ pub fn parser() -> impl Parser<Token, ast::UntypedDefinition, Error = ParseError
                     public,
                     return_annotation,
                     return_type: (),
-                    can_error: true,
+                    on_test_failure: ast::OnTestFailure::FailImmediately,
                 })
             },
         )

crates/aiken-lang/src/parser/definition/snapshots/def_invalid_property_test.snap

@@ -54,6 +54,6 @@ Test(
         ),
         return_type: (),
         end_position: 38,
-        can_error: false,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/def_property_test.snap

@@ -43,6 +43,6 @@ Test(
         ),
         return_type: (),
         end_position: 40,
-        can_error: false,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/def_property_test_annotated_fuzzer.snap

@@ -50,6 +50,6 @@ Test(
         ),
         return_type: (),
         end_position: 38,
-        can_error: false,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/def_test.snap

@@ -23,6 +23,6 @@ Test(
         ),
         return_type: (),
         end_position: 22,
-        can_error: false,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/def_test_fail.snap

@@ -54,6 +54,6 @@ Test(
         ),
         return_type: (),
         end_position: 60,
-        can_error: true,
+        on_test_failure: SucceedEventually,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/double_validator.snap

@@ -56,7 +56,7 @@ Validator(
             return_annotation: None,
             return_type: (),
             end_position: 52,
-            can_error: true,
+            on_test_failure: FailImmediately,
         },
         other_fun: Some(
             Function {
@@ -97,7 +97,7 @@ Validator(
                 return_annotation: None,
                 return_type: (),
                 end_position: 88,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
         location: 0..9,

crates/aiken-lang/src/parser/definition/snapshots/function_assignment_only.snap

@@ -46,6 +46,6 @@ Fn(
         return_annotation: None,
         return_type: (),
         end_position: 27,
-        can_error: true,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/function_empty.snap

@@ -23,6 +23,6 @@ Fn(
         return_annotation: None,
         return_type: (),
         end_position: 14,
-        can_error: true,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/function_non_public.snap

@@ -23,6 +23,6 @@ Fn(
         return_annotation: None,
         return_type: (),
         end_position: 10,
-        can_error: true,
+        on_test_failure: FailImmediately,
     },
 )

crates/aiken-lang/src/parser/definition/snapshots/validator.snap

@@ -56,7 +56,7 @@ Validator(
             return_annotation: None,
             return_type: (),
             end_position: 52,
-            can_error: true,
+            on_test_failure: FailImmediately,
         },
         other_fun: None,
         location: 0..9,

crates/aiken-lang/src/parser/definition/test.rs

@@ -1,5 +1,6 @@
 use crate::{
     ast,
+    ast::OnTestFailure,
     expr::UntypedExpr,
     parser::{
         annotation,
@@ -12,41 +13,42 @@ use crate::{
 use chumsky::prelude::*;
 
 pub fn parser() -> impl Parser<Token, ast::UntypedDefinition, Error = ParseError> {
-    // TODO: can remove Token::Bang after a few releases (curr v1.0.11)
-    just(Token::Bang)
-        .ignored()
-        .or_not()
-        .then_ignore(just(Token::Test))
-        .then(select! {Token::Name {name} => name})
+    just(Token::Test)
+        .ignore_then(select! {Token::Name {name} => name})
         .then(
             via()
                 .separated_by(just(Token::Comma))
                 .allow_trailing()
                 .delimited_by(just(Token::LeftParen), just(Token::RightParen)),
         )
-        .then(just(Token::Fail).ignored().or_not())
+        .then(
+            just(Token::Fail)
+                .ignore_then(just(Token::Once).ignored().or_not().map(|once| {
+                    once.map(|_| OnTestFailure::SucceedImmediately)
+                        .unwrap_or(OnTestFailure::SucceedEventually)
+                }))
+                .or_not(),
+        )
         .map_with_span(|name, span| (name, span))
         .then(
             expr::sequence()
                 .or_not()
                 .delimited_by(just(Token::LeftBrace), just(Token::RightBrace)),
         )
-        .map_with_span(
-            |(((((old_fail, name), arguments), fail), span_end), body), span| {
-                ast::UntypedDefinition::Test(ast::Function {
-                    arguments,
-                    body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)),
-                    doc: None,
-                    location: span_end,
-                    end_position: span.end - 1,
-                    name,
-                    public: false,
-                    return_annotation: Some(ast::Annotation::boolean(span)),
-                    return_type: (),
-                    can_error: fail.is_some() || old_fail.is_some(),
-                })
-            },
-        )
+        .map_with_span(|((((name, arguments), fail), span_end), body), span| {
+            ast::UntypedDefinition::Test(ast::Function {
+                arguments,
+                body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)),
+                doc: None,
+                location: span_end,
+                end_position: span.end - 1,
+                name,
+                public: false,
+                return_annotation: Some(ast::Annotation::boolean(span)),
+                return_type: (),
+                on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately),
+            })
+        })
 }
 
 pub fn via() -> impl Parser<Token, ast::UntypedArgVia, Error = ParseError> {

crates/aiken-lang/src/parser/lexer.rs

@@ -223,6 +223,7 @@ pub fn lexer() -> impl Parser<char, Vec<(Token, Span)>, Error = ParseError> {
         // TODO: remove this in a future release
         "error" => Token::Fail,
         "fail" => Token::Fail,
+        "once" => Token::Once,
         "as" => Token::As,
         "and" => Token::And,
         "or" => Token::Or,

crates/aiken-lang/src/parser/token.rs

@@ -78,6 +78,7 @@ pub enum Token {
     If,
     Else,
     Fail,
+    Once,
     Expect,
     Is,
     Let,
@@ -178,6 +179,7 @@ impl fmt::Display for Token {
             Token::Type => "type",
             Token::Test => "test",
             Token::Fail => "fail",
+            Token::Once => "once",
             Token::Validator => "validator",
             Token::Via => "via",
         };

crates/aiken-lang/src/snapshots/function_ambiguous_sequence.snap

@@ -49,7 +49,7 @@ Module {
                 return_annotation: None,
                 return_type: (),
                 end_position: 34,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
         Fn(
@@ -94,7 +94,7 @@ Module {
                 return_annotation: None,
                 return_type: (),
                 end_position: 71,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
         Fn(
@@ -141,7 +141,7 @@ Module {
                 return_annotation: None,
                 return_type: (),
                 end_position: 104,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
         Fn(
@@ -221,7 +221,7 @@ Module {
                 return_annotation: None,
                 return_type: (),
                 end_position: 154,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
     ],

crates/aiken-lang/src/snapshots/parse_unicode_offset_1.snap

@@ -51,7 +51,7 @@ Module {
                 return_annotation: None,
                 return_type: (),
                 end_position: 31,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
     ],

crates/aiken-lang/src/snapshots/parse_unicode_offset_2.snap

@@ -49,7 +49,7 @@ Module {
                 return_annotation: None,
                 return_type: (),
                 end_position: 29,
-                can_error: true,
+                on_test_failure: FailImmediately,
             },
         ),
     ],

crates/aiken-lang/src/tipo/environment.rs

@@ -218,7 +218,7 @@ impl<'a> Environment<'a> {
                 return_annotation,
                 return_type,
                 end_position,
-                can_error,
+                on_test_failure,
             }) => {
                 // Lookup the inferred function information
                 let function = self
@@ -263,7 +263,7 @@ impl<'a> Environment<'a> {
                     return_type,
                     body,
                     end_position,
-                    can_error,
+                    on_test_failure,
                 })
             }
             Definition::Validator(Validator {

crates/aiken-lang/src/tipo/expr.rs

@@ -55,7 +55,7 @@ pub(crate) fn infer_function(
         body,
         return_annotation,
         end_position,
-        can_error,
+        on_test_failure,
         return_type: _,
     } = fun;
 
@@ -174,7 +174,7 @@ pub(crate) fn infer_function(
             .return_type()
             .expect("Could not find return type for fn"),
         body,
-        can_error: *can_error,
+        on_test_failure: on_test_failure.clone(),
         end_position: *end_position,
     };
 

crates/aiken-lang/src/tipo/infer.rs

@@ -442,7 +442,7 @@ fn infer_definition(
                 return_annotation: typed_f.return_annotation,
                 return_type: typed_f.return_type,
                 body: typed_f.body,
-                can_error: typed_f.can_error,
+                on_test_failure: typed_f.on_test_failure,
                 end_position: typed_f.end_position,
             }))
         }

crates/aiken-project/src/telemetry.rs

@@ -2,7 +2,7 @@ use crate::{
     pretty,
     test_framework::{PropertyTestResult, TestResult, UnitTestResult},
 };
-use aiken_lang::{expr::UntypedExpr, format::Formatter};
+use aiken_lang::{ast::OnTestFailure, expr::UntypedExpr, format::Formatter};
 use owo_colors::{OwoColorize, Stream::Stderr};
 use std::{collections::BTreeMap, fmt::Display, path::PathBuf};
 use uplc::machine::cost_model::ExBudget;
@@ -338,7 +338,14 @@ fn fmt_test(
         }) if !result.is_success() => {
             test = format!(
                 "{test}\n{}",
-                assertion.to_string(Stderr, unit_test.can_error),
+                assertion.to_string(
+                    Stderr,
+                    match unit_test.on_test_failure {
+                        OnTestFailure::FailImmediately => false,
+                        OnTestFailure::SucceedEventually | OnTestFailure::SucceedImmediately =>
+                            true,
+                    }
+                ),
             );
         }
         _ => (),

crates/aiken-project/src/test_framework.rs

@@ -1,4 +1,5 @@
-use aiken_lang::{
+use aiken_lang::ast::OnTestFailure;
+pub(crate) use aiken_lang::{
     ast::{Arg, BinOp, DataTypeKey, IfBranch, Span, TypedDataType, TypedTest},
     builtins::bool,
     expr::{TypedExpr, UntypedExpr},
@@ -88,7 +89,7 @@ impl Test {
             name: test.name,
             program,
             assertion,
-            can_error: test.can_error,
+            on_test_failure: test.on_test_failure,
         })
     }
 
@@ -96,7 +97,7 @@ impl Test {
         input_path: PathBuf,
         module: String,
         name: String,
-        can_error: bool,
+        on_test_failure: OnTestFailure,
         program: Program<Name>,
         fuzzer: Fuzzer<Name>,
     ) -> Test {
@@ -105,7 +106,7 @@ impl Test {
             module,
             name,
             program,
-            can_error,
+            on_test_failure,
             fuzzer,
         })
     }
@@ -145,7 +146,7 @@ impl Test {
                 input_path,
                 module_name,
                 test.name,
-                test.can_error,
+                test.on_test_failure,
                 program,
                 Fuzzer {
                     program: fuzzer,
@@ -164,7 +165,7 @@ pub struct UnitTest {
     pub input_path: PathBuf,
     pub module: String,
     pub name: String,
-    pub can_error: bool,
+    pub on_test_failure: OnTestFailure,
     pub program: Program<Name>,
     pub assertion: Option<Assertion<(Constant, Rc<Type>)>>,
 }
@@ -177,7 +178,10 @@ impl UnitTest {
             .unwrap()
             .eval_version(ExBudget::max(), &plutus_version.into());
 
-        let success = !eval_result.failed(self.can_error);
+        let success = !eval_result.failed(match self.on_test_failure {
+            OnTestFailure::SucceedEventually | OnTestFailure::SucceedImmediately => true,
+            OnTestFailure::FailImmediately => false,
+        });
 
         TestResult::UnitTestResult(UnitTestResult {
             success,
@@ -196,7 +200,7 @@ pub struct PropertyTest {
     pub input_path: PathBuf,
     pub module: String,
     pub name: String,
-    pub can_error: bool,
+    pub on_test_failure: OnTestFailure,
     pub program: Program<Name>,
     pub fuzzer: Fuzzer<Name>,
 }
@@ -250,7 +254,7 @@ impl PropertyTest {
                     .filter(|s| PropertyTest::extract_label(s).is_none())
                     .collect(),
                 Ok(Some(counterexample.value)),
-                n - remaining + 1,
+                n - remaining,
             ),
             Err(FuzzerError { traces, uplc_error }) => (
                 traces
@@ -258,7 +262,7 @@ impl PropertyTest {
                     .filter(|s| PropertyTest::extract_label(s).is_none())
                     .collect(),
                 Err(uplc_error),
-                0,
+                n - remaining + 1,
             ),
         };
 
@@ -295,6 +299,8 @@ impl PropertyTest {
         labels: &mut BTreeMap<String, usize>,
         plutus_version: &'a PlutusVersion,
     ) -> Result<(Prng, Option<Counterexample<'a>>), FuzzerError> {
+        use OnTestFailure::*;
+
         let (next_prng, value) = prng
             .sample(&self.fuzzer.program)?
             .expect("A seeded PRNG returned 'None' which indicates a fuzzer is ill-formed and implemented wrongly; please contact library's authors.");
@@ -313,10 +319,16 @@ impl PropertyTest {
             }
         }
 
-        // NOTE: We do NOT pass self.can_error here, because when searching for
-        // failing properties, we do want to _keep running_ until we find a
-        // a failing case. It may not occur on the first run.
-        if result.failed(false) {
+        let is_failure = result.failed(false);
+
+        let is_success = !is_failure;
+
+        let keep_counterexample = match self.on_test_failure {
+            FailImmediately | SucceedImmediately => is_failure,
+            SucceedEventually => is_success,
+        };
+
+        if keep_counterexample {
             let mut counterexample = Counterexample {
                 value,
                 choices: next_prng.choices(),
@@ -327,16 +339,24 @@ impl PropertyTest {
                         Ok(Some((_, value))) => {
                             let result = self.eval(&value, plutus_version);
 
-                            let is_failure = result.failed(self.can_error);
+                            let is_failure = result.failed(false);
 
-                            let expect_failure = self.can_error;
+                            match self.on_test_failure {
+                                FailImmediately | SucceedImmediately => {
+                                    if is_failure {
+                                        Status::Keep(value)
+                                    } else {
+                                        Status::Ignore
+                                    }
+                                }
 
-                            // If the test no longer fails, it isn't better as we're only
-                            // interested in counterexamples.
-                            if (expect_failure && is_failure) || (!expect_failure && !is_failure) {
-                                Status::Ignore
-                            } else {
-                                Status::Keep(value)
+                                SucceedEventually => {
+                                    if is_failure {
+                                        Status::Ignore
+                                    } else {
+                                        Status::Keep(value)
+                                    }
+                                }
                             }
                         }
                     }
@@ -883,13 +903,12 @@ impl<U, T> TestResult<U, T> {
                 counterexample: Ok(counterexample),
                 test,
                 ..
-            }) => {
-                if test.can_error {
-                    counterexample.is_some()
-                } else {
+            }) => match test.on_test_failure {
+                OnTestFailure::FailImmediately | OnTestFailure::SucceedEventually => {
                     counterexample.is_none()
                 }
-            }
+                OnTestFailure::SucceedImmediately => counterexample.is_some(),
+            },
         }
     }