diff --git a/CHANGELOG.md b/CHANGELOG.md index 0330d546..be3dcc1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v1.0.29-alpha - UNRELEASED + +### Changed + +- **aiken-lang**: the keyword `fail` on property-based test semantic has changed and now consider a test to succeed only if **every** execution of the test failed (instead of just one). The previous behavior can be recovered by adding the keyword `once` after `fail`. @KtorZ + ## v1.0.28-alpha - 2024-05-23 ### Added diff --git a/crates/aiken-lang/src/ast.rs b/crates/aiken-lang/src/ast.rs index d4e2d6ec..fecbef99 100644 --- a/crates/aiken-lang/src/ast.rs +++ b/crates/aiken-lang/src/ast.rs @@ -223,6 +223,13 @@ pub type UntypedFunction = Function<(), UntypedExpr, UntypedArg>; pub type TypedTest = Function, TypedExpr, TypedArgVia>; pub type UntypedTest = Function<(), UntypedExpr, UntypedArgVia>; +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub enum OnTestFailure { + FailImmediately, + SucceedImmediately, + SucceedEventually, +} + #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Function { pub arguments: Vec, @@ -234,7 +241,7 @@ pub struct Function { pub return_annotation: Option, pub return_type: T, pub end_position: usize, - pub can_error: bool, + pub on_test_failure: OnTestFailure, } impl TypedFunction { @@ -279,7 +286,7 @@ impl From for UntypedFunction { return_annotation: f.return_annotation, return_type: f.return_type, body: f.body, - can_error: f.can_error, + on_test_failure: f.on_test_failure, end_position: f.end_position, } } @@ -296,7 +303,7 @@ impl From for TypedFunction { return_annotation: f.return_annotation, return_type: f.return_type, body: f.body, - can_error: f.can_error, + on_test_failure: f.on_test_failure, end_position: f.end_position, } } diff --git a/crates/aiken-lang/src/builtins.rs b/crates/aiken-lang/src/builtins.rs index 37cd9175..d5131909 100644 --- a/crates/aiken-lang/src/builtins.rs +++ b/crates/aiken-lang/src/builtins.rs @@ -1,7 +1,7 @@ use crate::{ ast::{ Annotation, Arg, ArgName, CallArg, DataTypeKey, Function, FunctionAccessKey, ModuleKind, - Span, TypedDataType, TypedFunction, UnOp, + OnTestFailure, Span, TypedDataType, TypedFunction, UnOp, }, expr::TypedExpr, tipo::{ @@ -944,7 +944,7 @@ pub fn prelude_functions(id_gen: &IdGenerator) -> IndexMap IndexMap IndexMap IndexMap Formatter<'comments> { arguments: args, body, end_position, - can_error, + on_test_failure, .. - }) => self.definition_test(name, args, body, *end_position, *can_error), + }) => self.definition_test(name, args, body, *end_position, on_test_failure), Definition::TypeAlias(TypeAlias { alias, @@ -547,14 +547,18 @@ impl<'comments> Formatter<'comments> { args: &'a [UntypedArgVia], body: &'a UntypedExpr, end_location: usize, - can_error: bool, + on_test_failure: &'a OnTestFailure, ) -> Document<'a> { // Fn name and args let head = "test " .to_doc() .append(name) .append(wrap_args(args.iter().map(|e| (self.fn_arg_via(e), false)))) - .append(if can_error { " fail" } else { "" }) + .append(match on_test_failure { + OnTestFailure::FailImmediately => "", + OnTestFailure::SucceedEventually => " fail", + OnTestFailure::SucceedImmediately => " fail once", + }) .group(); // Format body diff --git a/crates/aiken-lang/src/parser/definition/function.rs b/crates/aiken-lang/src/parser/definition/function.rs index 92d8af38..bfd68006 100644 --- a/crates/aiken-lang/src/parser/definition/function.rs +++ b/crates/aiken-lang/src/parser/definition/function.rs @@ -1,10 +1,9 @@ -use chumsky::prelude::*; - use crate::{ ast, expr::UntypedExpr, parser::{annotation, error::ParseError, expr, token::Token, utils}, }; +use chumsky::prelude::*; pub fn parser() -> impl Parser { utils::optional_flag(Token::Pub) @@ -41,7 +40,7 @@ pub fn parser() -> impl Parser impl Parser { - // TODO: can remove Token::Bang after a few releases (curr v1.0.11) - just(Token::Bang) - .ignored() - .or_not() - .then_ignore(just(Token::Test)) - .then(select! {Token::Name {name} => name}) + just(Token::Test) + .ignore_then(select! {Token::Name {name} => name}) .then( via() .separated_by(just(Token::Comma)) .allow_trailing() .delimited_by(just(Token::LeftParen), just(Token::RightParen)), ) - .then(just(Token::Fail).ignored().or_not()) + .then( + just(Token::Fail) + .ignore_then(just(Token::Once).ignored().or_not().map(|once| { + once.map(|_| OnTestFailure::SucceedImmediately) + .unwrap_or(OnTestFailure::SucceedEventually) + })) + .or_not(), + ) .map_with_span(|name, span| (name, span)) .then( expr::sequence() .or_not() .delimited_by(just(Token::LeftBrace), just(Token::RightBrace)), ) - .map_with_span( - |(((((old_fail, name), arguments), fail), span_end), body), span| { - ast::UntypedDefinition::Test(ast::Function { - arguments, - body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)), - doc: None, - location: span_end, - end_position: span.end - 1, - name, - public: false, - return_annotation: Some(ast::Annotation::boolean(span)), - return_type: (), - can_error: fail.is_some() || old_fail.is_some(), - }) - }, - ) + .map_with_span(|((((name, arguments), fail), span_end), body), span| { + ast::UntypedDefinition::Test(ast::Function { + arguments, + body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)), + doc: None, + location: span_end, + end_position: span.end - 1, + name, + public: false, + return_annotation: Some(ast::Annotation::boolean(span)), + return_type: (), + on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately), + }) + }) } pub fn via() -> impl Parser { diff --git a/crates/aiken-lang/src/parser/lexer.rs b/crates/aiken-lang/src/parser/lexer.rs index c1fc1282..097ec4fd 100644 --- a/crates/aiken-lang/src/parser/lexer.rs +++ b/crates/aiken-lang/src/parser/lexer.rs @@ -223,6 +223,7 @@ pub fn lexer() -> impl Parser, Error = ParseError> { // TODO: remove this in a future release "error" => Token::Fail, "fail" => Token::Fail, + "once" => Token::Once, "as" => Token::As, "and" => Token::And, "or" => Token::Or, diff --git a/crates/aiken-lang/src/parser/token.rs b/crates/aiken-lang/src/parser/token.rs index 6169749f..8f1fdbfa 100644 --- a/crates/aiken-lang/src/parser/token.rs +++ b/crates/aiken-lang/src/parser/token.rs @@ -78,6 +78,7 @@ pub enum Token { If, Else, Fail, + Once, Expect, Is, Let, @@ -178,6 +179,7 @@ impl fmt::Display for Token { Token::Type => "type", Token::Test => "test", Token::Fail => "fail", + Token::Once => "once", Token::Validator => "validator", Token::Via => "via", }; diff --git a/crates/aiken-lang/src/snapshots/function_ambiguous_sequence.snap b/crates/aiken-lang/src/snapshots/function_ambiguous_sequence.snap index 4113a500..3f757f9d 100644 --- a/crates/aiken-lang/src/snapshots/function_ambiguous_sequence.snap +++ b/crates/aiken-lang/src/snapshots/function_ambiguous_sequence.snap @@ -49,7 +49,7 @@ Module { return_annotation: None, return_type: (), end_position: 34, - can_error: true, + on_test_failure: FailImmediately, }, ), Fn( @@ -94,7 +94,7 @@ Module { return_annotation: None, return_type: (), end_position: 71, - can_error: true, + on_test_failure: FailImmediately, }, ), Fn( @@ -141,7 +141,7 @@ Module { return_annotation: None, return_type: (), end_position: 104, - can_error: true, + on_test_failure: FailImmediately, }, ), Fn( @@ -221,7 +221,7 @@ Module { return_annotation: None, return_type: (), end_position: 154, - can_error: true, + on_test_failure: FailImmediately, }, ), ], diff --git a/crates/aiken-lang/src/snapshots/parse_unicode_offset_1.snap b/crates/aiken-lang/src/snapshots/parse_unicode_offset_1.snap index 8c257e71..29f96a21 100644 --- a/crates/aiken-lang/src/snapshots/parse_unicode_offset_1.snap +++ b/crates/aiken-lang/src/snapshots/parse_unicode_offset_1.snap @@ -51,7 +51,7 @@ Module { return_annotation: None, return_type: (), end_position: 31, - can_error: true, + on_test_failure: FailImmediately, }, ), ], diff --git a/crates/aiken-lang/src/snapshots/parse_unicode_offset_2.snap b/crates/aiken-lang/src/snapshots/parse_unicode_offset_2.snap index f4555fe9..f491a175 100644 --- a/crates/aiken-lang/src/snapshots/parse_unicode_offset_2.snap +++ b/crates/aiken-lang/src/snapshots/parse_unicode_offset_2.snap @@ -49,7 +49,7 @@ Module { return_annotation: None, return_type: (), end_position: 29, - can_error: true, + on_test_failure: FailImmediately, }, ), ], diff --git a/crates/aiken-lang/src/tipo/environment.rs b/crates/aiken-lang/src/tipo/environment.rs index 972df9ad..a5b6300a 100644 --- a/crates/aiken-lang/src/tipo/environment.rs +++ b/crates/aiken-lang/src/tipo/environment.rs @@ -218,7 +218,7 @@ impl<'a> Environment<'a> { return_annotation, return_type, end_position, - can_error, + on_test_failure, }) => { // Lookup the inferred function information let function = self @@ -263,7 +263,7 @@ impl<'a> Environment<'a> { return_type, body, end_position, - can_error, + on_test_failure, }) } Definition::Validator(Validator { diff --git a/crates/aiken-lang/src/tipo/expr.rs b/crates/aiken-lang/src/tipo/expr.rs index 6c2a2ee9..f0b22562 100644 --- a/crates/aiken-lang/src/tipo/expr.rs +++ b/crates/aiken-lang/src/tipo/expr.rs @@ -55,7 +55,7 @@ pub(crate) fn infer_function( body, return_annotation, end_position, - can_error, + on_test_failure, return_type: _, } = fun; @@ -174,7 +174,7 @@ pub(crate) fn infer_function( .return_type() .expect("Could not find return type for fn"), body, - can_error: *can_error, + on_test_failure: on_test_failure.clone(), end_position: *end_position, }; diff --git a/crates/aiken-lang/src/tipo/infer.rs b/crates/aiken-lang/src/tipo/infer.rs index 51c70828..f4f21e85 100644 --- a/crates/aiken-lang/src/tipo/infer.rs +++ b/crates/aiken-lang/src/tipo/infer.rs @@ -442,7 +442,7 @@ fn infer_definition( return_annotation: typed_f.return_annotation, return_type: typed_f.return_type, body: typed_f.body, - can_error: typed_f.can_error, + on_test_failure: typed_f.on_test_failure, end_position: typed_f.end_position, })) } diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index bfc529c1..2b7da6de 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -2,7 +2,7 @@ use crate::{ pretty, test_framework::{PropertyTestResult, TestResult, UnitTestResult}, }; -use aiken_lang::{expr::UntypedExpr, format::Formatter}; +use aiken_lang::{ast::OnTestFailure, expr::UntypedExpr, format::Formatter}; use owo_colors::{OwoColorize, Stream::Stderr}; use std::{collections::BTreeMap, fmt::Display, path::PathBuf}; use uplc::machine::cost_model::ExBudget; @@ -338,7 +338,14 @@ fn fmt_test( }) if !result.is_success() => { test = format!( "{test}\n{}", - assertion.to_string(Stderr, unit_test.can_error), + assertion.to_string( + Stderr, + match unit_test.on_test_failure { + OnTestFailure::FailImmediately => false, + OnTestFailure::SucceedEventually | OnTestFailure::SucceedImmediately => + true, + } + ), ); } _ => (), diff --git a/crates/aiken-project/src/test_framework.rs b/crates/aiken-project/src/test_framework.rs index 43ba4bfe..305c76c4 100644 --- a/crates/aiken-project/src/test_framework.rs +++ b/crates/aiken-project/src/test_framework.rs @@ -1,4 +1,5 @@ -use aiken_lang::{ +use aiken_lang::ast::OnTestFailure; +pub(crate) use aiken_lang::{ ast::{Arg, BinOp, DataTypeKey, IfBranch, Span, TypedDataType, TypedTest}, builtins::bool, expr::{TypedExpr, UntypedExpr}, @@ -88,7 +89,7 @@ impl Test { name: test.name, program, assertion, - can_error: test.can_error, + on_test_failure: test.on_test_failure, }) } @@ -96,7 +97,7 @@ impl Test { input_path: PathBuf, module: String, name: String, - can_error: bool, + on_test_failure: OnTestFailure, program: Program, fuzzer: Fuzzer, ) -> Test { @@ -105,7 +106,7 @@ impl Test { module, name, program, - can_error, + on_test_failure, fuzzer, }) } @@ -145,7 +146,7 @@ impl Test { input_path, module_name, test.name, - test.can_error, + test.on_test_failure, program, Fuzzer { program: fuzzer, @@ -164,7 +165,7 @@ pub struct UnitTest { pub input_path: PathBuf, pub module: String, pub name: String, - pub can_error: bool, + pub on_test_failure: OnTestFailure, pub program: Program, pub assertion: Option)>>, } @@ -177,7 +178,10 @@ impl UnitTest { .unwrap() .eval_version(ExBudget::max(), &plutus_version.into()); - let success = !eval_result.failed(self.can_error); + let success = !eval_result.failed(match self.on_test_failure { + OnTestFailure::SucceedEventually | OnTestFailure::SucceedImmediately => true, + OnTestFailure::FailImmediately => false, + }); TestResult::UnitTestResult(UnitTestResult { success, @@ -196,7 +200,7 @@ pub struct PropertyTest { pub input_path: PathBuf, pub module: String, pub name: String, - pub can_error: bool, + pub on_test_failure: OnTestFailure, pub program: Program, pub fuzzer: Fuzzer, } @@ -250,7 +254,7 @@ impl PropertyTest { .filter(|s| PropertyTest::extract_label(s).is_none()) .collect(), Ok(Some(counterexample.value)), - n - remaining + 1, + n - remaining, ), Err(FuzzerError { traces, uplc_error }) => ( traces @@ -258,7 +262,7 @@ impl PropertyTest { .filter(|s| PropertyTest::extract_label(s).is_none()) .collect(), Err(uplc_error), - 0, + n - remaining + 1, ), }; @@ -295,6 +299,8 @@ impl PropertyTest { labels: &mut BTreeMap, plutus_version: &'a PlutusVersion, ) -> Result<(Prng, Option>), FuzzerError> { + use OnTestFailure::*; + let (next_prng, value) = prng .sample(&self.fuzzer.program)? .expect("A seeded PRNG returned 'None' which indicates a fuzzer is ill-formed and implemented wrongly; please contact library's authors."); @@ -313,10 +319,16 @@ impl PropertyTest { } } - // NOTE: We do NOT pass self.can_error here, because when searching for - // failing properties, we do want to _keep running_ until we find a - // a failing case. It may not occur on the first run. - if result.failed(false) { + let is_failure = result.failed(false); + + let is_success = !is_failure; + + let keep_counterexample = match self.on_test_failure { + FailImmediately | SucceedImmediately => is_failure, + SucceedEventually => is_success, + }; + + if keep_counterexample { let mut counterexample = Counterexample { value, choices: next_prng.choices(), @@ -327,16 +339,24 @@ impl PropertyTest { Ok(Some((_, value))) => { let result = self.eval(&value, plutus_version); - let is_failure = result.failed(self.can_error); + let is_failure = result.failed(false); - let expect_failure = self.can_error; + match self.on_test_failure { + FailImmediately | SucceedImmediately => { + if is_failure { + Status::Keep(value) + } else { + Status::Ignore + } + } - // If the test no longer fails, it isn't better as we're only - // interested in counterexamples. - if (expect_failure && is_failure) || (!expect_failure && !is_failure) { - Status::Ignore - } else { - Status::Keep(value) + SucceedEventually => { + if is_failure { + Status::Ignore + } else { + Status::Keep(value) + } + } } } } @@ -883,13 +903,12 @@ impl TestResult { counterexample: Ok(counterexample), test, .. - }) => { - if test.can_error { - counterexample.is_some() - } else { + }) => match test.on_test_failure { + OnTestFailure::FailImmediately | OnTestFailure::SucceedEventually => { counterexample.is_none() } - } + OnTestFailure::SucceedImmediately => counterexample.is_some(), + }, } }