From 782c3274f881211c415b3c971d195c4ad534d563 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:17:14 +0100 Subject: [PATCH 01/15] remove unnecessary intermediate variables Introduced in some previous commits, so basically reverting that. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 806b8ec8..f9c4df5b 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -372,9 +372,7 @@ impl PropertyTest { let mut counterexample = None; while *remaining > 0 && counterexample.is_none() { - let (next_prng, cex) = self.run_once(prng, labels, plutus_version)?; - prng = next_prng; - counterexample = cex; + (prng, counterexample) = self.run_once(prng, labels, plutus_version)?; *remaining -= 1; } From d87e7f808fcf609a8e21c39968b9b4b046a02ccd Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:17:42 +0100 Subject: [PATCH 02/15] remove duplicate entry in CHANGELOG likely due to an wrong merge conflict resolution. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- CHANGELOG.md | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202d09c1..d64386d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,34 +16,6 @@ - **aiken-lang**: `write_bits` can now be used from aiken/builtins. @Microproofs -### Changed - -- **aiken-project**: The `aiken.toml` file no longer supports `v1` and `v2` for the plutus version field. @rvcas -- **aiken-project**: `Error::TomlLoading` now looks much better - [see](https://github.com/aiken-lang/aiken/issues/1032#issuecomment-2562122101). @rvcas -- **aiken-lang**: 10-20% optimization improvements via case-constr, rearranging function definitions (while maintaining dependency ordering), - and allowing inlining in if_then_else_error cases which preserve the same error semantics for a program. @Microproofs - -### Fixed - -- **aiken**: panic error when using `aiken uplc decode` on cbor encoded flat bytes. @rvcas -- **aiken-lang**: comment formatting in pipelines leading to confusion. @rvcas -- **aiken-lang**: preserve holes discard name in function captures (see [#1080](https://github.com/aiken-lang/aiken/issues/1080)). @KtorZ -- **uplc**: Added deserialization match for the new builtin indices. - -## v1.1.11 - UNRELEASED - -### Added - -- **aiken**: support for `bench` keyword to define benchmarks. @Riley-Kilgore - -## v1.1.10 - 2025-01-21 - -### Added - -- **aiken-project**: `export` output now supports the functions `return_type`. @rvcas -- **aiken-lang**: `write_bits` can now be used from aiken/builtins. @Microproofs - - ### Changed - **aiken-project**: The `aiken.toml` file no longer supports `v1` and `v2` for the plutus version field. @rvcas From 497f663513165854b9b2f62391c3dbe12d6d0919 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:19:21 +0100 Subject: [PATCH 03/15] actually fail if a (seeded) sampler return None This is not supposed to happen, as only replayed sampler/fuzzer can stop. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index f9c4df5b..cb5e955b 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -531,8 +531,9 @@ impl Benchmark { } Ok(None) => { - break; + panic!("A seeded PRNG returned 'None' which indicates a sampler is ill-formed and implemented wrongly; please contact library's authors."); } + Err(e) => { results.push(BenchmarkResult { test: self.clone(), From 0a4d60b8216bf69780c3d12fd5e699f4eed239b1 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:20:22 +0100 Subject: [PATCH 04/15] minor aesthetic changes in test framework. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index cb5e955b..33305d47 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -539,7 +539,7 @@ impl Benchmark { test: self.clone(), cost: ExBudget::default(), success: false, - traces: vec![format!("Fuzzer error: {}", e)], + traces: vec![e.to_string()], }); break; } @@ -1133,8 +1133,8 @@ impl TestResult { pub fn traces(&self) -> &[String] { match self { TestResult::UnitTestResult(UnitTestResult { traces, .. }) - | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) => traces, - TestResult::Benchmark(BenchmarkResult { traces, .. }) => traces, + | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) + | TestResult::Benchmark(BenchmarkResult { traces, .. }) => traces, } } } From 428d30c3bb04fd639c1423cdf2baa722b8be9145 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:20:41 +0100 Subject: [PATCH 05/15] refactor and fix benchmark type-checking Fixes: - Do not allow bench with no arguments; this causes a compiler panic down the line otherwise. - Do not force the return value to be a boolean or void. We do not actually control what's returned by benchmark, so anything really works here. Refactor: - Re-use code between test and bench type-checking; especially the bits related to gathering information about the via arguments. There's quite a lot and simply copy-pasting everything will likely cause issues and discrepency at the first change. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/tipo/error.rs | 9 + crates/aiken-lang/src/tipo/infer.rs | 277 +++++++++++----------------- 2 files changed, 117 insertions(+), 169 deletions(-) diff --git a/crates/aiken-lang/src/tipo/error.rs b/crates/aiken-lang/src/tipo/error.rs index 0ca7f295..e2a414d4 100644 --- a/crates/aiken-lang/src/tipo/error.rs +++ b/crates/aiken-lang/src/tipo/error.rs @@ -317,6 +317,14 @@ You can use '{discard}' and numbers to distinguish between similar names. location: Span, }, + #[error("I notice a benchmark definition without any argument.\n")] + #[diagnostic(url("https://aiken-lang.org/language-tour/bench"))] + #[diagnostic(code("arity::bench"))] + IncorrectBenchmarkArity { + #[label("must have exactly one argument")] + location: Span, + }, + #[error( "I saw {} field{} in a context where there should be {}.\n", given.if_supports_color(Stdout, |s| s.purple()), @@ -1158,6 +1166,7 @@ impl ExtraData for Error { | Error::UnknownPurpose { .. } | Error::UnknownValidatorHandler { .. } | Error::UnexpectedValidatorFallback { .. } + | Error::IncorrectBenchmarkArity { .. } | Error::MustInferFirst { .. } => None, Error::UnknownType { name, .. } diff --git a/crates/aiken-lang/src/tipo/infer.rs b/crates/aiken-lang/src/tipo/infer.rs index f5fb0a61..a8945811 100644 --- a/crates/aiken-lang/src/tipo/infer.rs +++ b/crates/aiken-lang/src/tipo/infer.rs @@ -12,7 +12,8 @@ use crate::{ TypedDefinition, TypedModule, TypedValidator, UntypedArg, UntypedDefinition, UntypedModule, UntypedPattern, UntypedValidator, Use, Validator, }, - expr::{TypedExpr, UntypedAssignmentKind}, + expr::{TypedExpr, UntypedAssignmentKind, UntypedExpr}, + parser::token::Token, tipo::{expr::infer_function, Span, Type, TypeVar}, IdGenerator, }; @@ -347,67 +348,8 @@ fn infer_definition( }); } - let typed_via = ExprTyper::new(environment, tracing).infer(arg.via.clone())?; - - let hydrator: &mut Hydrator = hydrators.get_mut(&f.name).unwrap(); - - let provided_inner_type = arg - .arg - .annotation - .as_ref() - .map(|ann| hydrator.type_from_annotation(ann, environment)) - .transpose()?; - - let (inferred_annotation, inferred_inner_type) = infer_fuzzer( - environment, - provided_inner_type.clone(), - &typed_via.tipo(), - &arg.via.location(), - )?; - - // Ensure that the annotation, if any, matches the type inferred from the - // Fuzzer. - if let Some(provided_inner_type) = provided_inner_type { - if !arg - .arg - .annotation - .as_ref() - .unwrap() - .is_logically_equal(&inferred_annotation) - { - return Err(Error::CouldNotUnify { - location: arg.arg.location, - expected: inferred_inner_type.clone(), - given: provided_inner_type.clone(), - situation: Some(UnifyErrorSituation::FuzzerAnnotationMismatch), - rigid_type_names: hydrator.rigid_names(), - }); - } - } - - // Replace the pre-registered type for the test function, to allow inferring - // the function body with the right type arguments. - let scope = environment - .scope - .get_mut(&f.name) - .expect("Could not find preregistered type for test"); - if let Type::Fn { - ref ret, - ref alias, - args: _, - } = scope.tipo.as_ref() - { - scope.tipo = Rc::new(Type::Fn { - ret: ret.clone(), - args: vec![inferred_inner_type.clone()], - alias: alias.clone(), - }) - } - - Ok(( - Some((typed_via, inferred_inner_type)), - Some(inferred_annotation), - )) + extract_via_information(&f, arg, hydrators, environment, tracing, infer_fuzzer) + .map(|(typed_via, annotation)| (Some(typed_via), Some(annotation))) } None => Ok((None, None)), }?; @@ -466,130 +408,50 @@ fn infer_definition( } Definition::Benchmark(f) => { + let err_incorrect_arity = || { + Err(Error::IncorrectBenchmarkArity { + location: f + .location + .map(|start, end| (start + Token::Benchmark.to_string().len() + 1, end)), + }) + }; + let (typed_via, annotation) = match f.arguments.first() { + None => return err_incorrect_arity(), Some(arg) => { if f.arguments.len() > 1 { - return Err(Error::IncorrectTestArity { - count: f.arguments.len(), - location: f - .arguments - .get(1) - .expect("arguments.len() > 1") - .arg - .location, - }); + return err_incorrect_arity(); } - let typed_via = ExprTyper::new(environment, tracing).infer(arg.via.clone())?; - - let hydrator: &mut Hydrator = hydrators.get_mut(&f.name).unwrap(); - - let provided_inner_type = arg - .arg - .annotation - .as_ref() - .map(|ann| hydrator.type_from_annotation(ann, environment)) - .transpose()?; - - let (inferred_annotation, inferred_inner_type) = infer_sampler( - environment, - provided_inner_type.clone(), - &typed_via.tipo(), - &arg.via.location(), - )?; - - // Ensure that the annotation, if any, matches the type inferred from the - // Sampler. - if let Some(provided_inner_type) = provided_inner_type { - if !arg - .arg - .annotation - .as_ref() - .unwrap() - .is_logically_equal(&inferred_annotation) - { - return Err(Error::CouldNotUnify { - location: arg.arg.location, - expected: inferred_inner_type.clone(), - given: provided_inner_type.clone(), - situation: Some(UnifyErrorSituation::SamplerAnnotationMismatch), - rigid_type_names: hydrator.rigid_names(), - }); - } - } - - // Replace the pre-registered type for the benchmark function, to allow inferring - // the function body with the right type arguments. - let scope = environment - .scope - .get_mut(&f.name) - .expect("Could not find preregistered type for benchmark"); - if let Type::Fn { - ref ret, - ref alias, - args: _, - } = scope.tipo.as_ref() - { - scope.tipo = Rc::new(Type::Fn { - ret: ret.clone(), - args: vec![inferred_inner_type.clone()], - alias: alias.clone(), - }) - } - - Ok(( - Some((typed_via, inferred_inner_type)), - Some(inferred_annotation), - )) + extract_via_information(&f, arg, hydrators, environment, tracing, infer_sampler) } - None => Ok((None, None)), }?; let typed_f = infer_function(&f.into(), module_name, hydrators, environment, tracing)?; - let is_bool = environment.unify( - typed_f.return_type.clone(), - Type::bool(), - typed_f.location, - false, - ); + let arguments = { + let arg = typed_f + .arguments + .first() + .expect("has exactly one argument") + .to_owned(); - let is_void = environment.unify( - typed_f.return_type.clone(), - Type::void(), - typed_f.location, - false, - ); - - if is_bool.or(is_void).is_err() { - return Err(Error::IllegalTestType { - location: typed_f.location, - }); - } + vec![ArgVia { + arg: TypedArg { + tipo: typed_via.1, + annotation: Some(annotation), + ..arg + }, + via: typed_via.0, + }] + }; Ok(Definition::Benchmark(Function { doc: typed_f.doc, location: typed_f.location, name: typed_f.name, public: typed_f.public, - arguments: match typed_via { - Some((via, tipo)) => { - let arg = typed_f - .arguments - .first() - .expect("has exactly one argument") - .to_owned(); - vec![ArgVia { - arg: TypedArg { - tipo, - annotation, - ..arg - }, - via, - }] - } - None => vec![], - }, + arguments, return_annotation: typed_f.return_annotation, return_type: typed_f.return_type, body: typed_f.body, @@ -823,6 +685,83 @@ fn infer_definition( } } +#[allow(clippy::result_large_err)] +fn extract_via_information( + f: &Function<(), UntypedExpr, ArgVia>, + arg: &ArgVia, + hydrators: &mut HashMap, + environment: &mut Environment<'_>, + tracing: Tracing, + infer_via: F, +) -> Result<((TypedExpr, Rc), Annotation), Error> +where + F: FnOnce( + &mut Environment<'_>, + Option>, + &Rc, + &Span, + ) -> Result<(Annotation, Rc), Error>, +{ + let typed_via = ExprTyper::new(environment, tracing).infer(arg.via.clone())?; + + let hydrator: &mut Hydrator = hydrators.get_mut(&f.name).unwrap(); + + let provided_inner_type = arg + .arg + .annotation + .as_ref() + .map(|ann| hydrator.type_from_annotation(ann, environment)) + .transpose()?; + + let (inferred_annotation, inferred_inner_type) = infer_via( + environment, + provided_inner_type.clone(), + &typed_via.tipo(), + &arg.via.location(), + )?; + + // Ensure that the annotation, if any, matches the type inferred from the + // Fuzzer. + if let Some(provided_inner_type) = provided_inner_type { + if !arg + .arg + .annotation + .as_ref() + .unwrap() + .is_logically_equal(&inferred_annotation) + { + return Err(Error::CouldNotUnify { + location: arg.arg.location, + expected: inferred_inner_type.clone(), + given: provided_inner_type.clone(), + situation: Some(UnifyErrorSituation::FuzzerAnnotationMismatch), + rigid_type_names: hydrator.rigid_names(), + }); + } + } + + // Replace the pre-registered type for the test function, to allow inferring + // the function body with the right type arguments. + let scope = environment + .scope + .get_mut(&f.name) + .expect("Could not find preregistered type for test"); + if let Type::Fn { + ref ret, + ref alias, + args: _, + } = scope.tipo.as_ref() + { + scope.tipo = Rc::new(Type::Fn { + ret: ret.clone(), + args: vec![inferred_inner_type.clone()], + alias: alias.clone(), + }) + } + + Ok(((typed_via, inferred_inner_type), inferred_annotation)) +} + #[allow(clippy::result_large_err)] fn infer_fuzzer( environment: &mut Environment<'_>, From 2a1253cb52aa69eaf1b2e8fd458c97dfb7e7e51d Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:23:34 +0100 Subject: [PATCH 06/15] Add additional test to check for Sampler alias formatting. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/tipo/pretty.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/aiken-lang/src/tipo/pretty.rs b/crates/aiken-lang/src/tipo/pretty.rs index 1aa81ea5..017ecea2 100644 --- a/crates/aiken-lang/src/tipo/pretty.rs +++ b/crates/aiken-lang/src/tipo/pretty.rs @@ -685,6 +685,7 @@ mod tests { }), "Identity Bool>", ); + assert_string!(Type::sampler(Type::int()), "Sampler"); } #[test] From 37f721ff0637cc48d86985531c59901a6737d46b Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:31:58 +0100 Subject: [PATCH 07/15] fixup aesthetics Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 18 +++++++++--------- crates/aiken-project/src/error.rs | 2 +- crates/aiken-project/src/lib.rs | 2 +- crates/aiken-project/src/telemetry.rs | 2 +- crates/aiken-project/src/telemetry/json.rs | 10 +++++----- crates/aiken-project/src/telemetry/terminal.rs | 6 +++--- crates/aiken-project/src/test_framework.rs | 5 +++-- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 33305d47..85131125 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -523,7 +523,7 @@ impl Benchmark { prng = new_prng; let mut eval_result = self.eval(&value, plutus_version); results.push(BenchmarkResult { - test: self.clone(), + bench: self.clone(), cost: eval_result.cost(), success: true, traces: eval_result.logs().to_vec(), @@ -536,7 +536,7 @@ impl Benchmark { Err(e) => { results.push(BenchmarkResult { - test: self.clone(), + bench: self.clone(), cost: ExBudget::default(), success: false, traces: vec![e.to_string()], @@ -1068,7 +1068,7 @@ where pub enum TestResult { UnitTestResult(UnitTestResult), PropertyTestResult(PropertyTestResult), - Benchmark(BenchmarkResult), + BenchmarkResult(BenchmarkResult), } unsafe impl Send for TestResult {} @@ -1083,7 +1083,7 @@ impl TestResult<(Constant, Rc), PlutusData> { TestResult::PropertyTestResult(test) => { TestResult::PropertyTestResult(test.reify(data_types)) } - TestResult::Benchmark(result) => TestResult::Benchmark(result), + TestResult::BenchmarkResult(result) => TestResult::BenchmarkResult(result), } } } @@ -1106,7 +1106,7 @@ impl TestResult { } OnTestFailure::SucceedImmediately => counterexample.is_some(), }, - TestResult::Benchmark(BenchmarkResult { success, .. }) => *success, + TestResult::BenchmarkResult(BenchmarkResult { success, .. }) => *success, } } @@ -1116,7 +1116,7 @@ impl TestResult { TestResult::PropertyTestResult(PropertyTestResult { ref test, .. }) => { test.module.as_str() } - TestResult::Benchmark(BenchmarkResult { ref test, .. }) => test.module.as_str(), + TestResult::BenchmarkResult(BenchmarkResult { ref bench, .. }) => bench.module.as_str(), } } @@ -1126,7 +1126,7 @@ impl TestResult { TestResult::PropertyTestResult(PropertyTestResult { ref test, .. }) => { test.name.as_str() } - TestResult::Benchmark(BenchmarkResult { ref test, .. }) => test.name.as_str(), + TestResult::BenchmarkResult(BenchmarkResult { ref bench, .. }) => bench.name.as_str(), } } @@ -1134,7 +1134,7 @@ impl TestResult { match self { TestResult::UnitTestResult(UnitTestResult { traces, .. }) | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) - | TestResult::Benchmark(BenchmarkResult { traces, .. }) => traces, + | TestResult::BenchmarkResult(BenchmarkResult { traces, .. }) => traces, } } } @@ -1472,7 +1472,7 @@ impl Assertion { #[derive(Debug, Clone)] pub struct BenchmarkResult { - pub test: Benchmark, + pub bench: Benchmark, pub cost: ExBudget, pub success: bool, pub traces: Vec, diff --git a/crates/aiken-project/src/error.rs b/crates/aiken-project/src/error.rs index 97e99aea..656e19f8 100644 --- a/crates/aiken-project/src/error.rs +++ b/crates/aiken-project/src/error.rs @@ -193,7 +193,7 @@ impl Error { test.input_path.to_path_buf(), test.program.to_pretty(), ), - TestResult::Benchmark(_) => ("bench".to_string(), PathBuf::new(), String::new()), // todo + TestResult::BenchmarkResult(_) => ("bench".to_string(), PathBuf::new(), String::new()), // todo }; Error::TestFailure { diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index e9d41bb3..a1c14719 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -1135,7 +1135,7 @@ where Test::Benchmark(benchmark) => benchmark .benchmark(seed, property_max_success, plutus_version) .into_iter() - .map(TestResult::Benchmark) + .map(TestResult::BenchmarkResult) .collect::>(), }) .collect::), PlutusData>>>() diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index 6e49b7c9..94491c43 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -134,7 +134,7 @@ pub(crate) fn find_max_execution_units(xs: &[TestResult]) -> (usize, us (max_mem, max_cpu, max_iter) } } - TestResult::Benchmark(..) => { + TestResult::BenchmarkResult(..) => { unreachable!("property returned benchmark result ?!") } }); diff --git a/crates/aiken-project/src/telemetry/json.rs b/crates/aiken-project/src/telemetry/json.rs index 4b9f807a..b72e7b87 100644 --- a/crates/aiken-project/src/telemetry/json.rs +++ b/crates/aiken-project/src/telemetry/json.rs @@ -43,10 +43,10 @@ impl EventListener for Json { let benchmark_results: Vec<_> = tests .into_iter() .filter_map(|test| { - if let TestResult::Benchmark(result) = test { + if let TestResult::BenchmarkResult(result) = test { Some(serde_json::json!({ - "name": result.test.name, - "module": result.test.module, + "name": result.bench.name, + "module": result.bench.module, "memory": result.cost.mem, "cpu": result.cost.cpu })) @@ -74,7 +74,7 @@ fn fmt_test_json(result: &TestResult) -> serde_json::V TestResult::PropertyTestResult(PropertyTestResult { ref test, .. }) => { &test.on_test_failure } - TestResult::Benchmark(_) => unreachable!("benchmark returned in JSON output"), + TestResult::BenchmarkResult(_) => unreachable!("benchmark returned in JSON output"), }; let mut test = json!({ @@ -120,7 +120,7 @@ fn fmt_test_json(result: &TestResult) -> serde_json::V Err(err) => json!({"error": err.to_string()}), }; } - TestResult::Benchmark(_) => unreachable!("benchmark returned in JSON output"), + TestResult::BenchmarkResult(_) => unreachable!("benchmark returned in JSON output"), } if !result.traces().is_empty() { diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index c234f944..d0dfd698 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -226,8 +226,8 @@ impl EventListener for Terminal { } Event::FinishedBenchmarks { tests, .. } => { for test in tests { - if let TestResult::Benchmark(result) = test { - println!("{} {} ", result.test.name.bold(), "BENCH".blue(),); + if let TestResult::BenchmarkResult(result) = test { + println!("{} {} ", result.bench.name.bold(), "BENCH".blue(),); println!(" Memory: {} bytes", result.cost.mem); println!(" CPU: {} units", result.cost.cpu); println!(); @@ -292,7 +292,7 @@ fn fmt_test( if *iterations > 1 { "s" } else { "" } ); } - TestResult::Benchmark(benchmark) => { + TestResult::BenchmarkResult(benchmark) => { let mem_pad = pretty::pad_left(benchmark.cost.mem.to_string(), max_mem, " "); let cpu_pad = pretty::pad_left(benchmark.cost.cpu.to_string(), max_cpu, " "); diff --git a/crates/aiken-project/src/test_framework.rs b/crates/aiken-project/src/test_framework.rs index bf5f744b..f8e2dc7f 100644 --- a/crates/aiken-project/src/test_framework.rs +++ b/crates/aiken-project/src/test_framework.rs @@ -278,7 +278,9 @@ mod test { PropertyTest::DEFAULT_MAX_SUCCESS, &PlutusVersion::default(), ) { - TestResult::UnitTestResult(..) => unreachable!("property returned unit-test result ?!"), + TestResult::BenchmarkResult(..) | TestResult::UnitTestResult(..) => { + unreachable!("property returned non-property result ?!") + } TestResult::PropertyTestResult(result) => { assert!( result @@ -289,7 +291,6 @@ mod test { result.labels ) } - TestResult::Benchmark(..) => unreachable!("property returned benchmark result ?!"), } } From a7f4ecef9d7aff66859c7b35a7785bf7ef1a6810 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:58:57 +0100 Subject: [PATCH 08/15] more aesthetic changes. In particular, using a concrete enum instead of a string to avoid an unnecessary incomplete pattern-match, and remove superfluous comments. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-project/src/lib.rs | 37 ++++++++++++------- crates/aiken-project/src/telemetry.rs | 2 +- crates/aiken-project/src/telemetry/json.rs | 4 +- .../aiken-project/src/telemetry/terminal.rs | 6 +-- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index a1c14719..0f8e7c6c 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -83,6 +83,12 @@ enum AddModuleBy { Path(PathBuf), } +#[derive(Debug, Clone, Copy)] +enum Runnable { + Test, + Bench, +} + pub struct Project where T: EventListener, @@ -471,28 +477,30 @@ where seed, times_to_run, } => { + let verbose = false; + let tests = - self.collect_benchmarks(false, match_tests, exact_match, options.tracing)?; + self.collect_benchmarks(verbose, match_tests, exact_match, options.tracing)?; if !tests.is_empty() { self.event_listener.handle_event(Event::RunningBenchmarks); } - let tests = self.run_benchmarks(tests, seed, times_to_run); + let benchmarks = self.run_benchmarks(tests, seed, times_to_run); - let errors: Vec = tests + let errors: Vec = benchmarks .iter() .filter_map(|e| { if e.is_success() { None } else { - Some(Error::from_test_result(e, false)) + Some(Error::from_test_result(e, verbose)) } }) .collect(); self.event_listener - .handle_event(Event::FinishedBenchmarks { seed, tests }); + .handle_event(Event::FinishedBenchmarks { seed, benchmarks }); if !errors.is_empty() { Err(errors) @@ -954,7 +962,7 @@ where fn collect_test_items( &mut self, - kind: &str, // "test" or "bench" + kind: Runnable, verbose: bool, match_tests: Option>, exact_match: bool, @@ -993,8 +1001,8 @@ where for def in checked_module.ast.definitions() { let func = match (kind, def) { - ("test", Definition::Test(func)) => Some(func), - ("bench", Definition::Benchmark(func)) => Some(func), + (Runnable::Test, Definition::Test(func)) => Some(func), + (Runnable::Bench, Definition::Benchmark(func)) => Some(func), _ => None, }; @@ -1049,19 +1057,18 @@ where } tests.push(match kind { - "test" => Test::from_function_definition( + Runnable::Test => Test::from_function_definition( &mut generator, test.to_owned(), module_name, input_path, ), - "bench" => Test::from_benchmark_definition( + Runnable::Bench => Test::from_benchmark_definition( &mut generator, test.to_owned(), module_name, input_path, ), - _ => unreachable!("Invalid test kind"), }); } @@ -1075,7 +1082,7 @@ where exact_match: bool, tracing: Tracing, ) -> Result, Error> { - self.collect_test_items("test", verbose, match_tests, exact_match, tracing) + self.collect_test_items(Runnable::Test, verbose, match_tests, exact_match, tracing) } fn collect_benchmarks( @@ -1085,7 +1092,7 @@ where exact_match: bool, tracing: Tracing, ) -> Result, Error> { - self.collect_test_items("bench", verbose, match_tests, exact_match, tracing) + self.collect_test_items(Runnable::Bench, verbose, match_tests, exact_match, tracing) } fn run_tests( @@ -1107,7 +1114,9 @@ where Test::PropertyTest(property_test) => { property_test.run(seed, property_max_success, plutus_version) } - Test::Benchmark(_) => unreachable!("Benchmarks cannot be run in PBT."), + Test::Benchmark(_) => { + unreachable!("found unexpected benchmark amongst collected tests.") + } }) .collect::), PlutusData>>>() .into_iter() diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index 94491c43..0e3e7d19 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -50,7 +50,7 @@ pub enum Event { }, FinishedBenchmarks { seed: u32, - tests: Vec>, + benchmarks: Vec>, }, WaitingForBuildDirLock, ResolvingPackages { diff --git a/crates/aiken-project/src/telemetry/json.rs b/crates/aiken-project/src/telemetry/json.rs index b72e7b87..ee76b0f5 100644 --- a/crates/aiken-project/src/telemetry/json.rs +++ b/crates/aiken-project/src/telemetry/json.rs @@ -39,8 +39,8 @@ impl EventListener for Json { }); println!("{}", serde_json::to_string_pretty(&json_output).unwrap()); } - Event::FinishedBenchmarks { tests, seed } => { - let benchmark_results: Vec<_> = tests + Event::FinishedBenchmarks { benchmarks, seed } => { + let benchmark_results: Vec<_> = benchmarks .into_iter() .filter_map(|test| { if let TestResult::BenchmarkResult(result) = test { diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index d0dfd698..16e2578e 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -224,9 +224,9 @@ impl EventListener for Terminal { "...".if_supports_color(Stderr, |s| s.bold()) ); } - Event::FinishedBenchmarks { tests, .. } => { - for test in tests { - if let TestResult::BenchmarkResult(result) = test { + Event::FinishedBenchmarks { benchmarks, .. } => { + for bench in benchmarks { + if let TestResult::BenchmarkResult(result) = bench { println!("{} {} ", result.bench.name.bold(), "BENCH".blue(),); println!(" Memory: {} bytes", result.cost.mem); println!(" CPU: {} units", result.cost.cpu); From 2dbc33e91f12e97d857b9954e6ebbf7cd956874f Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 17:40:07 +0100 Subject: [PATCH 09/15] fuse together bench & test runners, and collect all bench measures. This commit removes some duplication between bench and test runners, as well as fixing the results coming out of running benchmarks. Running benchmarks is expected to yield multiple measures, for each of the iteration. For now, it'll suffice to show results for each size; but eventually, we'll possibly try to interpolate results with different curves and pick the best candidate. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 123 +++++++++--------- crates/aiken-project/src/lib.rs | 120 ++++++----------- crates/aiken-project/src/options.rs | 4 +- crates/aiken-project/src/telemetry.rs | 2 +- crates/aiken-project/src/telemetry/json.rs | 9 +- .../aiken-project/src/telemetry/terminal.rs | 26 +--- crates/aiken-project/src/test_framework.rs | 41 +++--- 7 files changed, 134 insertions(+), 191 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 85131125..cf0b66e3 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -28,6 +28,12 @@ use uplc::{ }; use vec1::{vec1, Vec1}; +#[derive(Debug, Clone, Copy)] +pub enum RunnableKind { + Test, + Bench, +} + /// ----- Test ----------------------------------------------------------------- /// /// Aiken supports two kinds of tests: unit and property. A unit test is a simply @@ -117,15 +123,15 @@ impl Test { }) } - pub fn from_test_definition( + pub fn from_function_definition( generator: &mut CodeGenerator<'_>, test: TypedTest, module_name: String, input_path: PathBuf, - is_benchmark: bool, + kind: RunnableKind, ) -> Test { if test.arguments.is_empty() { - if is_benchmark { + if matches!(kind, RunnableKind::Bench) { unreachable!("benchmark must have at least one argument"); } else { Self::unit_test(generator, test, module_name, input_path) @@ -153,8 +159,8 @@ impl Test { // apply onto it later. let generator_program = generator.clone().generate_raw(&via, &[], &module_name); - if is_benchmark { - Test::Benchmark(Benchmark { + match kind { + RunnableKind::Bench => Test::Benchmark(Benchmark { input_path, module: module_name, name: test.name, @@ -165,9 +171,8 @@ impl Test { type_info, stripped_type_info, }, - }) - } else { - Self::property_test( + }), + RunnableKind::Test => Self::property_test( input_path, module_name, test.name, @@ -178,27 +183,26 @@ impl Test { stripped_type_info, type_info, }, - ) + ), } } } - pub fn from_benchmark_definition( - generator: &mut CodeGenerator<'_>, - test: TypedTest, - module_name: String, - input_path: PathBuf, - ) -> Test { - Self::from_test_definition(generator, test, module_name, input_path, true) - } - - pub fn from_function_definition( - generator: &mut CodeGenerator<'_>, - test: TypedTest, - module_name: String, - input_path: PathBuf, - ) -> Test { - Self::from_test_definition(generator, test, module_name, input_path, false) + pub fn run( + self, + seed: u32, + max_success: usize, + plutus_version: &PlutusVersion, + ) -> TestResult<(Constant, Rc), PlutusData> { + match self { + Test::UnitTest(unit_test) => TestResult::UnitTestResult(unit_test.run(plutus_version)), + Test::PropertyTest(property_test) => { + TestResult::PropertyTestResult(property_test.run(seed, max_success, plutus_version)) + } + Test::Benchmark(benchmark) => { + TestResult::BenchmarkResult(benchmark.run(seed, max_success, plutus_version)) + } + } } } @@ -217,7 +221,7 @@ pub struct UnitTest { unsafe impl Send for UnitTest {} impl UnitTest { - pub fn run(self, plutus_version: &PlutusVersion) -> TestResult<(Constant, Rc), T> { + pub fn run(self, plutus_version: &PlutusVersion) -> UnitTestResult<(Constant, Rc)> { let mut eval_result = Program::::try_from(self.program.clone()) .unwrap() .eval_version(ExBudget::max(), &plutus_version.into()); @@ -233,13 +237,13 @@ impl UnitTest { } traces.extend(eval_result.logs()); - TestResult::UnitTestResult(UnitTestResult { + UnitTestResult { success, test: self.to_owned(), spent_budget: eval_result.cost(), traces, assertion: self.assertion, - }) + } } } @@ -317,12 +321,12 @@ impl PropertyTest { /// Run a property test from a given seed. The property is run at most DEFAULT_MAX_SUCCESS times. It /// may stops earlier on failure; in which case a 'counterexample' is returned. - pub fn run( + pub fn run( self, seed: u32, n: usize, plutus_version: &PlutusVersion, - ) -> TestResult { + ) -> PropertyTestResult { let mut labels = BTreeMap::new(); let mut remaining = n; @@ -352,13 +356,13 @@ impl PropertyTest { ), }; - TestResult::PropertyTestResult(PropertyTestResult { + PropertyTestResult { test: self, counterexample, iterations, labels, traces, - }) + } } pub fn run_n_times<'a>( @@ -503,51 +507,43 @@ pub struct Benchmark { unsafe impl Send for Benchmark {} impl Benchmark { - pub fn benchmark( + pub fn run( self, seed: u32, max_iterations: usize, plutus_version: &PlutusVersion, - ) -> Vec { - let mut results = Vec::with_capacity(max_iterations); + ) -> BenchmarkResult { + let mut measures = Vec::with_capacity(max_iterations); let mut iteration = 0; let mut prng = Prng::from_seed(seed); + let mut success = true; + + while success && max_iterations > iteration { + let size = Data::integer(num_bigint::BigInt::from(iteration as i64)); + let fuzzer = self.sampler.program.apply_data(size); - while max_iterations > iteration { - let fuzzer = self - .sampler - .program - .apply_data(Data::integer(num_bigint::BigInt::from(iteration as i64))); match prng.sample(&fuzzer) { - Ok(Some((new_prng, value))) => { - prng = new_prng; - let mut eval_result = self.eval(&value, plutus_version); - results.push(BenchmarkResult { - bench: self.clone(), - cost: eval_result.cost(), - success: true, - traces: eval_result.logs().to_vec(), - }); - } - Ok(None) => { panic!("A seeded PRNG returned 'None' which indicates a sampler is ill-formed and implemented wrongly; please contact library's authors."); } - Err(e) => { - results.push(BenchmarkResult { - bench: self.clone(), - cost: ExBudget::default(), - success: false, - traces: vec![e.to_string()], - }); - break; + Ok(Some((new_prng, value))) => { + prng = new_prng; + measures.push(self.eval(&value, plutus_version).cost()) + } + + Err(_e) => { + success = false; } } iteration += 1; } - results + BenchmarkResult { + bench: self, + measures, + success, + } } pub fn eval(&self, value: &PlutusData, plutus_version: &PlutusVersion) -> EvalResult { @@ -1133,8 +1129,8 @@ impl TestResult { pub fn traces(&self) -> &[String] { match self { TestResult::UnitTestResult(UnitTestResult { traces, .. }) - | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) - | TestResult::BenchmarkResult(BenchmarkResult { traces, .. }) => traces, + | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) => traces, + TestResult::BenchmarkResult(BenchmarkResult { .. }) => &[], } } } @@ -1473,9 +1469,8 @@ impl Assertion { #[derive(Debug, Clone)] pub struct BenchmarkResult { pub bench: Benchmark, - pub cost: ExBudget, + pub measures: Vec, pub success: bool, - pub traces: Vec, } unsafe impl Send for BenchmarkResult {} diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index 0f8e7c6c..9e343be5 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -40,7 +40,7 @@ use aiken_lang::{ format::{Formatter, MAX_COLUMNS}, gen_uplc::CodeGenerator, line_numbers::LineNumbers, - test_framework::{Test, TestResult}, + test_framework::{RunnableKind, Test, TestResult}, tipo::{Type, TypeInfo}, utils, IdGenerator, }; @@ -83,12 +83,6 @@ enum AddModuleBy { Path(PathBuf), } -#[derive(Debug, Clone, Copy)] -enum Runnable { - Test, - Bench, -} - pub struct Project where T: EventListener, @@ -305,20 +299,20 @@ where pub fn benchmark( &mut self, - match_tests: Option>, + match_benchmarks: Option>, exact_match: bool, seed: u32, - times_to_run: usize, + iterations: usize, env: Option, ) -> Result<(), Vec> { let options = Options { tracing: Tracing::silent(), env, code_gen_mode: CodeGenMode::Benchmark { - match_tests, + match_benchmarks, exact_match, seed, - times_to_run, + iterations, }, blueprint_path: self.blueprint_path(None), }; @@ -438,7 +432,7 @@ where self.event_listener.handle_event(Event::RunningTests); } - let tests = self.run_tests(tests, seed, property_max_success); + let tests = self.run_runnables(tests, seed, property_max_success); self.checks_count = if tests.is_empty() { None @@ -472,21 +466,25 @@ where } } CodeGenMode::Benchmark { - match_tests, + match_benchmarks, exact_match, seed, - times_to_run, + iterations, } => { let verbose = false; - let tests = - self.collect_benchmarks(verbose, match_tests, exact_match, options.tracing)?; + let benchmarks = self.collect_benchmarks( + verbose, + match_benchmarks, + exact_match, + options.tracing, + )?; - if !tests.is_empty() { + if !benchmarks.is_empty() { self.event_listener.handle_event(Event::RunningBenchmarks); } - let benchmarks = self.run_benchmarks(tests, seed, times_to_run); + let benchmarks = self.run_runnables(benchmarks, seed, iterations); let errors: Vec = benchmarks .iter() @@ -962,7 +960,7 @@ where fn collect_test_items( &mut self, - kind: Runnable, + kind: RunnableKind, verbose: bool, match_tests: Option>, exact_match: bool, @@ -1001,8 +999,8 @@ where for def in checked_module.ast.definitions() { let func = match (kind, def) { - (Runnable::Test, Definition::Test(func)) => Some(func), - (Runnable::Bench, Definition::Benchmark(func)) => Some(func), + (RunnableKind::Test, Definition::Test(func)) => Some(func), + (RunnableKind::Bench, Definition::Benchmark(func)) => Some(func), _ => None, }; @@ -1056,20 +1054,13 @@ where }) } - tests.push(match kind { - Runnable::Test => Test::from_function_definition( - &mut generator, - test.to_owned(), - module_name, - input_path, - ), - Runnable::Bench => Test::from_benchmark_definition( - &mut generator, - test.to_owned(), - module_name, - input_path, - ), - }); + tests.push(Test::from_function_definition( + &mut generator, + test.to_owned(), + module_name, + input_path, + kind, + )); } Ok(tests) @@ -1082,7 +1073,13 @@ where exact_match: bool, tracing: Tracing, ) -> Result, Error> { - self.collect_test_items(Runnable::Test, verbose, match_tests, exact_match, tracing) + self.collect_test_items( + RunnableKind::Test, + verbose, + match_tests, + exact_match, + tracing, + ) } fn collect_benchmarks( @@ -1092,14 +1089,20 @@ where exact_match: bool, tracing: Tracing, ) -> Result, Error> { - self.collect_test_items(Runnable::Bench, verbose, match_tests, exact_match, tracing) + self.collect_test_items( + RunnableKind::Bench, + verbose, + match_tests, + exact_match, + tracing, + ) } - fn run_tests( + fn run_runnables( &self, tests: Vec, seed: u32, - property_max_success: usize, + max_success: usize, ) -> Vec> { use rayon::prelude::*; @@ -1109,44 +1112,7 @@ where tests .into_par_iter() - .map(|test| match test { - Test::UnitTest(unit_test) => unit_test.run(plutus_version), - Test::PropertyTest(property_test) => { - property_test.run(seed, property_max_success, plutus_version) - } - Test::Benchmark(_) => { - unreachable!("found unexpected benchmark amongst collected tests.") - } - }) - .collect::), PlutusData>>>() - .into_iter() - .map(|test| test.reify(&data_types)) - .collect() - } - - fn run_benchmarks( - &self, - tests: Vec, - seed: u32, - property_max_success: usize, - ) -> Vec> { - use rayon::prelude::*; - - let data_types = utils::indexmap::as_ref_values(&self.data_types); - let plutus_version = &self.config.plutus; - - tests - .into_par_iter() - .flat_map(|test| match test { - Test::UnitTest(_) | Test::PropertyTest(_) => { - unreachable!("Tests cannot be ran during benchmarking.") - } - Test::Benchmark(benchmark) => benchmark - .benchmark(seed, property_max_success, plutus_version) - .into_iter() - .map(TestResult::BenchmarkResult) - .collect::>(), - }) + .map(|test| test.run(seed, max_success, plutus_version)) .collect::), PlutusData>>>() .into_iter() .map(|test| test.reify(&data_types)) diff --git a/crates/aiken-project/src/options.rs b/crates/aiken-project/src/options.rs index 2afa69ed..8b4fdf9b 100644 --- a/crates/aiken-project/src/options.rs +++ b/crates/aiken-project/src/options.rs @@ -30,10 +30,10 @@ pub enum CodeGenMode { }, Build(bool), Benchmark { - match_tests: Option>, + match_benchmarks: Option>, exact_match: bool, seed: u32, - times_to_run: usize, + iterations: usize, }, NoOp, } diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index 0e3e7d19..5ef77d13 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -135,7 +135,7 @@ pub(crate) fn find_max_execution_units(xs: &[TestResult]) -> (usize, us } } TestResult::BenchmarkResult(..) => { - unreachable!("property returned benchmark result ?!") + unreachable!("unexpected benchmark found amongst test results.") } }); diff --git a/crates/aiken-project/src/telemetry/json.rs b/crates/aiken-project/src/telemetry/json.rs index ee76b0f5..e9c37623 100644 --- a/crates/aiken-project/src/telemetry/json.rs +++ b/crates/aiken-project/src/telemetry/json.rs @@ -47,8 +47,13 @@ impl EventListener for Json { Some(serde_json::json!({ "name": result.bench.name, "module": result.bench.module, - "memory": result.cost.mem, - "cpu": result.cost.cpu + "measures": result.measures + .into_iter() + .map(|measure| serde_json::json!({ + "memory": measure.mem, + "cpu": measure.cpu + })) + .collect::>() })) } else { None diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index 16e2578e..2f1adefd 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -224,15 +224,8 @@ impl EventListener for Terminal { "...".if_supports_color(Stderr, |s| s.bold()) ); } - Event::FinishedBenchmarks { benchmarks, .. } => { - for bench in benchmarks { - if let TestResult::BenchmarkResult(result) = bench { - println!("{} {} ", result.bench.name.bold(), "BENCH".blue(),); - println!(" Memory: {} bytes", result.cost.mem); - println!(" CPU: {} units", result.cost.cpu); - println!(); - } - } + Event::FinishedBenchmarks { .. } => { + eprintln!("TODO: FinishedBenchmarks"); } } } @@ -292,19 +285,8 @@ fn fmt_test( if *iterations > 1 { "s" } else { "" } ); } - TestResult::BenchmarkResult(benchmark) => { - let mem_pad = pretty::pad_left(benchmark.cost.mem.to_string(), max_mem, " "); - let cpu_pad = pretty::pad_left(benchmark.cost.cpu.to_string(), max_cpu, " "); - - test = format!( - "{test} [mem: {mem_unit}, cpu: {cpu_unit}]", - mem_unit = pretty::style_if(styled, mem_pad, |s| s - .if_supports_color(Stderr, |s| s.cyan()) - .to_string()), - cpu_unit = pretty::style_if(styled, cpu_pad, |s| s - .if_supports_color(Stderr, |s| s.cyan()) - .to_string()), - ); + TestResult::BenchmarkResult(..) => { + unreachable!("unexpected benchmark found amongst test results.") } } diff --git a/crates/aiken-project/src/test_framework.rs b/crates/aiken-project/src/test_framework.rs index f8e2dc7f..1cd509a1 100644 --- a/crates/aiken-project/src/test_framework.rs +++ b/crates/aiken-project/src/test_framework.rs @@ -101,6 +101,7 @@ mod test { test.to_owned(), module_name.to_string(), PathBuf::new(), + RunnableKind::Test, ), data_types, ) @@ -245,13 +246,12 @@ mod test { } "#}); - assert!(prop - .run::<()>( - 42, - PropertyTest::DEFAULT_MAX_SUCCESS, - &PlutusVersion::default() - ) - .is_success()); + assert!(TestResult::PropertyTestResult::<(), _>(prop.run( + 42, + PropertyTest::DEFAULT_MAX_SUCCESS, + &PlutusVersion::default() + )) + .is_success()); } #[test] @@ -273,25 +273,20 @@ mod test { } "#}); - match prop.run::<()>( + let result = prop.run( 42, PropertyTest::DEFAULT_MAX_SUCCESS, &PlutusVersion::default(), - ) { - TestResult::BenchmarkResult(..) | TestResult::UnitTestResult(..) => { - unreachable!("property returned non-property result ?!") - } - TestResult::PropertyTestResult(result) => { - assert!( - result - .labels - .iter() - .eq(vec![(&"head".to_string(), &53), (&"tail".to_string(), &47)]), - "labels: {:#?}", - result.labels - ) - } - } + ); + + assert!( + result + .labels + .iter() + .eq(vec![(&"head".to_string(), &53), (&"tail".to_string(), &47)]), + "labels: {:#?}", + result.labels + ); } #[test] From 41440f131b11d3ba549ac6cfe1cc45667fa4dfd8 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 18:26:02 +0100 Subject: [PATCH 10/15] rework sizing of benchmarks, taking measures at different points The idea is to get a good sample of measures from running benchmarks with various sizes, so one can get an idea of how well a function performs at various sizes. Given that size can be made arbitrarily large, and that we currently report all benchmarks, I installed a fibonacci heuristic to gather data points from 0 to the max size using an increasing stepping. Defined as a trait as I already anticipate we might need different sizing strategy, likely driven by the user via a command-line option; but for now, this will do. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 82 +++++++++++++++++++--- crates/aiken-project/src/telemetry/json.rs | 5 +- crates/aiken/src/cmd/benchmark.rs | 27 +++---- 3 files changed, 89 insertions(+), 25 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index cf0b66e3..f4881ac0 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -14,7 +14,7 @@ use pallas_primitives::alonzo::{Constr, PlutusData}; use patricia_tree::PatriciaMap; use std::{ borrow::Borrow, - collections::BTreeMap, + collections::{BTreeMap, VecDeque}, convert::TryFrom, fmt::{Debug, Display}, ops::Deref, @@ -506,21 +506,84 @@ pub struct Benchmark { unsafe impl Send for Benchmark {} +trait Sizer { + fn is_done(&self) -> bool; + fn next(&mut self) -> usize; +} + +struct FibonacciSizer { + max_size: usize, + previous_sizes: VecDeque, + current_size: usize, +} + +impl FibonacciSizer { + fn new(max_size: usize) -> Self { + Self { + max_size, + previous_sizes: VecDeque::new(), + current_size: 1, + } + } +} + +impl Sizer for FibonacciSizer { + fn is_done(&self) -> bool { + self.current_size >= self.max_size + } + + fn next(&mut self) -> usize { + match self.previous_sizes.len() { + 0 => { + self.previous_sizes.push_front(1); + return 0; + } + 1 => { + self.previous_sizes.push_front(1); + return 1; + } + _ => self.current_size += self.previous_sizes.pop_back().unwrap(), + } + + self.previous_sizes.push_front(self.current_size); + + self.current_size.min(self.max_size) + } +} + +#[cfg(test)] +mod test_sizer { + use super::{FibonacciSizer, Sizer}; + + #[test] + pub fn fib_sizer_sequence() { + let mut sizer = FibonacciSizer::new(100); + let mut sizes = Vec::new(); + while !sizer.is_done() { + sizes.push(sizer.next()) + } + assert_eq!(sizes, vec![0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 100]) + } +} + impl Benchmark { + pub const DEFAULT_MAX_SIZE: usize = 10; + pub fn run( self, seed: u32, - max_iterations: usize, + max_size: usize, plutus_version: &PlutusVersion, ) -> BenchmarkResult { - let mut measures = Vec::with_capacity(max_iterations); - let mut iteration = 0; + let mut measures = Vec::with_capacity(max_size); + let mut sizer = FibonacciSizer::new(max_size); let mut prng = Prng::from_seed(seed); let mut success = true; - while success && max_iterations > iteration { - let size = Data::integer(num_bigint::BigInt::from(iteration as i64)); - let fuzzer = self.sampler.program.apply_data(size); + while success && !sizer.is_done() { + let size = sizer.next(); + let size_as_data = Data::integer(num_bigint::BigInt::from(size)); + let fuzzer = self.sampler.program.apply_data(size_as_data); match prng.sample(&fuzzer) { Ok(None) => { @@ -529,14 +592,13 @@ impl Benchmark { Ok(Some((new_prng, value))) => { prng = new_prng; - measures.push(self.eval(&value, plutus_version).cost()) + measures.push((size, self.eval(&value, plutus_version).cost())) } Err(_e) => { success = false; } } - iteration += 1; } BenchmarkResult { @@ -1469,7 +1531,7 @@ impl Assertion { #[derive(Debug, Clone)] pub struct BenchmarkResult { pub bench: Benchmark, - pub measures: Vec, + pub measures: Vec<(usize, ExBudget)>, pub success: bool, } diff --git a/crates/aiken-project/src/telemetry/json.rs b/crates/aiken-project/src/telemetry/json.rs index e9c37623..f2f0893b 100644 --- a/crates/aiken-project/src/telemetry/json.rs +++ b/crates/aiken-project/src/telemetry/json.rs @@ -50,8 +50,9 @@ impl EventListener for Json { "measures": result.measures .into_iter() .map(|measure| serde_json::json!({ - "memory": measure.mem, - "cpu": measure.cpu + "size": measure.0, + "memory": measure.1.mem, + "cpu": measure.1.cpu })) .collect::>() })) diff --git a/crates/aiken/src/cmd/benchmark.rs b/crates/aiken/src/cmd/benchmark.rs index 65d7a60d..1a495f8d 100644 --- a/crates/aiken/src/cmd/benchmark.rs +++ b/crates/aiken/src/cmd/benchmark.rs @@ -1,4 +1,4 @@ -use aiken_lang::test_framework::PropertyTest; +use aiken_lang::test_framework::Benchmark; use aiken_project::watch::with_project; use rand::prelude::*; use std::{ @@ -17,18 +17,20 @@ pub struct Args { #[clap(long)] seed: Option, - /// How many times we will run each benchmark in the relevant project. - #[clap(long, default_value_t = PropertyTest::DEFAULT_MAX_SUCCESS)] - times_to_run: usize, + /// The maximum size to benchmark with. Note that this does not necessarily equates the number + /// of measurements actually performed but controls the maximum size given to a Sampler. + #[clap(long, default_value_t = Benchmark::DEFAULT_MAX_SIZE)] + max_size: usize, - /// Only run tests if they match any of these strings. + /// Only run benchmarks if they match any of these strings. + /// /// You can match a module with `-m aiken/list` or `-m list`. /// You can match a test with `-m "aiken/list.{map}"` or `-m "aiken/option.{flatten_1}"` #[clap(short, long)] - match_tests: Option>, + match_benchmarks: Option>, - /// This is meant to be used with `--match-tests`. - /// It forces test names to match exactly + /// This is meant to be used with `--match-benchmarks`. + /// It forces benchmark names to match exactly #[clap(short, long)] exact_match: bool, @@ -39,10 +41,10 @@ pub struct Args { pub fn exec( Args { directory, - match_tests, + match_benchmarks, exact_match, seed, - times_to_run, + max_size, env, }: Args, ) -> miette::Result<()> { @@ -55,12 +57,11 @@ pub fn exec( false, !io::stdout().is_terminal(), |p| { - // We don't want to check here, we want to benchmark p.benchmark( - match_tests.clone(), + match_benchmarks.clone(), exact_match, seed, - times_to_run, + max_size, env.clone(), ) }, From b4aa877d6ac60fbdb3c6d3f92a9464b1f78c3cde Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sun, 9 Feb 2025 15:21:45 +0100 Subject: [PATCH 11/15] rework benchmarks output Going for a terminal plot, for now, as this was the original idea and it is immediately visual. All benchmark points can also be obtained as JSON when redirecting the output, like for tests. So all-in-all, we provide a flexible output which should be useful. Whether it is the best we can do, time (and people/users) will tell. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- Cargo.lock | 439 +++++++++++------- crates/aiken-lang/src/test_framework.rs | 75 +-- crates/aiken-project/Cargo.toml | 2 + crates/aiken-project/src/telemetry.rs | 34 +- .../aiken-project/src/telemetry/terminal.rs | 139 +++++- 5 files changed, 431 insertions(+), 258 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89eb1f4d..ee03e888 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,7 +63,7 @@ dependencies = [ "ignore", "indoc", "inquire", - "miette 7.4.0", + "miette 7.5.0", "num-bigint", "openssl", "openssl-probe", @@ -94,7 +94,7 @@ dependencies = [ "indoc", "insta", "itertools 0.10.5", - "miette 7.4.0", + "miette 7.5.0", "num-bigint", "ordinal", "owo-colors 3.5.0", @@ -121,7 +121,7 @@ dependencies = [ "itertools 0.10.5", "lsp-server", "lsp-types", - "miette 7.4.0", + "miette 7.5.0", "owo-colors 3.5.0", "serde", "serde_json", @@ -151,7 +151,7 @@ dependencies = [ "insta", "itertools 0.10.5", "katex", - "miette 7.4.0", + "miette 7.5.0", "notify", "num-bigint", "owo-colors 3.5.0", @@ -167,10 +167,12 @@ dependencies = [ "rayon", "regex", "reqwest", + "rgb", "semver", "serde", "serde_json", "strip-ansi-escapes", + "textplots", "thiserror 1.0.69", "tokio", "toml", @@ -227,11 +229,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] @@ -267,7 +270,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -388,9 +391,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitvec" @@ -427,9 +430,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "serde", @@ -446,9 +449,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "bytemuck" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -458,9 +467,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "bzip2" @@ -491,9 +500,9 @@ checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" [[package]] name = "cc" -version = "1.2.5" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" dependencies = [ "jobserver", "libc", @@ -555,9 +564,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" dependencies = [ "clap_builder", "clap_derive", @@ -565,9 +574,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -580,23 +589,23 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.40" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2e663e3e3bed2d32d065a8404024dad306e699a04263ec59919529f803aee9" +checksum = "375f9d8255adeeedd51053574fd8d4ba875ea5fa558e86617b07f09f1680c8b6" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -623,7 +632,7 @@ dependencies = [ "nom", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -632,6 +641,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "console" version = "0.15.10" @@ -683,9 +702,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -775,9 +794,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-bigint" @@ -938,14 +957,24 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", +] + +[[package]] +name = "drawille" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64e461c3f1e69d99372620640b3fd5f0309eeda2e26e4af69f6760c0e1df845" +dependencies = [ + "colored", + "fnv", ] [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" [[package]] name = "ecdsa" @@ -1172,7 +1201,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -1224,7 +1253,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", ] [[package]] @@ -1239,7 +1280,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", "libgit2-sys", "log", @@ -1248,9 +1289,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "globset" @@ -1288,7 +1329,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -1393,9 +1434,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -1564,7 +1605,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -1622,9 +1663,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1683,24 +1724,25 @@ dependencies = [ [[package]] name = "insta" -version = "1.41.1" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9ffc4d4892617c50a928c52b2961cb5174b6fc6ebf252b2fac9d21955c48b8" +checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" dependencies = [ "console", - "lazy_static", "linked-hash-map", + "once_cell", "pest", "pest_derive", + "pin-project", "serde", "similar", ] [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is_ci" @@ -1749,9 +1791,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -1850,16 +1892,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", "redox_syscall", ] [[package]] name = "libz-sys" -version = "1.1.20" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" dependencies = [ "cc", "libc", @@ -1875,9 +1917,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -1897,9 +1939,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "lsp-server" @@ -1947,14 +1989,14 @@ dependencies = [ [[package]] name = "miette" -version = "7.4.0" +version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317f146e2eb7021892722af37cf1b971f0a70c8406f487e24952667616192c64" +checksum = "1a955165f87b37fd1862df2a59547ac542c77ef6d17c666f619d1ad22dd89484" dependencies = [ "backtrace", "backtrace-ext", "cfg-if", - "miette-derive 7.4.0", + "miette-derive 7.5.0", "owo-colors 4.1.0", "supports-color 3.0.2", "supports-hyperlinks", @@ -1973,18 +2015,18 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] name = "miette-derive" -version = "7.4.0" +version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c9b935fbe1d6cbd1dac857b54a688145e2d93f48db36010514d0f612d0ad67" +checksum = "bf45bf44ab49be92fd1227a3be6fc6f617f1a337c06af54981048574d8783147" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -2021,7 +2063,7 @@ checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -2032,9 +2074,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -2047,7 +2089,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -2058,15 +2100,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] [[package]] name = "native-tls" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +checksum = "0dab59f8e050d5df8e4dd87d9206fb6f65a483e20ac9fda365ade4fab353196c" dependencies = [ "libc", "log", @@ -2104,7 +2146,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -2172,17 +2214,17 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -2199,14 +2241,14 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" @@ -2219,9 +2261,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", @@ -2377,7 +2419,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31f2f4539bffe53fc4b4da301df49d114b845b077bd5727b7fe2bd9d8df2ae68" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -2432,7 +2474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.9", + "thiserror 2.0.11", "ucd-trie", ] @@ -2456,7 +2498,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -2477,14 +2519,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.7.1", +] + +[[package]] +name = "pin-project" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2558,9 +2620,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -2573,7 +2635,7 @@ checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.6.0", + "bitflags 2.8.0", "lazy_static", "num-traits", "rand", @@ -2600,7 +2662,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "memchr", "pulldown-cmark-escape", "unicase", @@ -2630,9 +2692,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2670,7 +2732,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -2708,7 +2770,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -2717,7 +2779,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.69", ] @@ -2801,6 +2863,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" +dependencies = [ + "bytemuck", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2809,11 +2880,11 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -2831,9 +2902,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rusty-fork" @@ -2849,9 +2920,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "same-file" @@ -2915,7 +2986,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation", "core-foundation-sys", "libc", @@ -2924,9 +2995,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -2934,40 +3005,40 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "memchr", "ryu", @@ -2982,7 +3053,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -3076,9 +3147,9 @@ dependencies = [ [[package]] name = "similar" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "slab" @@ -3174,7 +3245,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -3227,9 +3298,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.91" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -3250,7 +3321,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -3282,12 +3353,13 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3303,6 +3375,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "textplots" +version = "0.8.7" +source = "git+https://github.com/aiken-lang/textplots-rs.git#20d166340cdc41fc0a8d8f2390ce69b99e73e3ec" +dependencies = [ + "drawille", + "rgb", +] + [[package]] name = "textwrap" version = "0.16.1" @@ -3324,11 +3405,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.11", ] [[package]] @@ -3339,18 +3420,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -3393,9 +3474,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -3411,13 +3492,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -3470,7 +3551,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -3502,7 +3583,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -3552,9 +3633,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-linebreak" @@ -3692,9 +3773,9 @@ dependencies = [ [[package]] name = "wait-timeout" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", ] @@ -3725,35 +3806,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "wasm-bindgen" -version = "0.2.99" +name = "wasi" +version = "0.13.3+wasi-0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -3764,9 +3855,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3774,28 +3865,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3999,6 +4093,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -4052,7 +4155,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", "synstructure", ] @@ -4074,7 +4177,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -4094,7 +4197,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", "synstructure", ] @@ -4115,7 +4218,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] @@ -4137,7 +4240,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.98", ] [[package]] diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index f4881ac0..01887356 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -14,7 +14,7 @@ use pallas_primitives::alonzo::{Constr, PlutusData}; use patricia_tree::PatriciaMap; use std::{ borrow::Borrow, - collections::{BTreeMap, VecDeque}, + collections::BTreeMap, convert::TryFrom, fmt::{Debug, Display}, ops::Deref, @@ -506,66 +506,6 @@ pub struct Benchmark { unsafe impl Send for Benchmark {} -trait Sizer { - fn is_done(&self) -> bool; - fn next(&mut self) -> usize; -} - -struct FibonacciSizer { - max_size: usize, - previous_sizes: VecDeque, - current_size: usize, -} - -impl FibonacciSizer { - fn new(max_size: usize) -> Self { - Self { - max_size, - previous_sizes: VecDeque::new(), - current_size: 1, - } - } -} - -impl Sizer for FibonacciSizer { - fn is_done(&self) -> bool { - self.current_size >= self.max_size - } - - fn next(&mut self) -> usize { - match self.previous_sizes.len() { - 0 => { - self.previous_sizes.push_front(1); - return 0; - } - 1 => { - self.previous_sizes.push_front(1); - return 1; - } - _ => self.current_size += self.previous_sizes.pop_back().unwrap(), - } - - self.previous_sizes.push_front(self.current_size); - - self.current_size.min(self.max_size) - } -} - -#[cfg(test)] -mod test_sizer { - use super::{FibonacciSizer, Sizer}; - - #[test] - pub fn fib_sizer_sequence() { - let mut sizer = FibonacciSizer::new(100); - let mut sizes = Vec::new(); - while !sizer.is_done() { - sizes.push(sizer.next()) - } - assert_eq!(sizes, vec![0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 100]) - } -} - impl Benchmark { pub const DEFAULT_MAX_SIZE: usize = 10; @@ -576,14 +516,15 @@ impl Benchmark { plutus_version: &PlutusVersion, ) -> BenchmarkResult { let mut measures = Vec::with_capacity(max_size); - let mut sizer = FibonacciSizer::new(max_size); let mut prng = Prng::from_seed(seed); let mut success = true; + let mut size = 0; - while success && !sizer.is_done() { - let size = sizer.next(); - let size_as_data = Data::integer(num_bigint::BigInt::from(size)); - let fuzzer = self.sampler.program.apply_data(size_as_data); + while success && max_size >= size { + let fuzzer = self + .sampler + .program + .apply_term(&Term::Constant(Constant::Integer(size.into()).into())); match prng.sample(&fuzzer) { Ok(None) => { @@ -599,6 +540,8 @@ impl Benchmark { success = false; } } + + size += 1; } BenchmarkResult { diff --git a/crates/aiken-project/Cargo.toml b/crates/aiken-project/Cargo.toml index 4c8dfdd3..30cccd28 100644 --- a/crates/aiken-project/Cargo.toml +++ b/crates/aiken-project/Cargo.toml @@ -42,10 +42,12 @@ pulldown-cmark = { version = "0.12.0", default-features = false, features = [ rayon = "1.7.0" regex = "1.7.1" reqwest = { version = "0.11.14", features = ["blocking", "json"] } +rgb = "0.8.50" semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = { version = "1.0.94", features = ["preserve_order"] } strip-ansi-escapes = "0.1.1" +textplots = { git = "https://github.com/aiken-lang/textplots-rs.git" } thiserror = "1.0.39" tokio = { version = "1.26.0", features = ["full"] } toml = "0.7.2" diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index 5ef77d13..221d8bfd 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -1,6 +1,6 @@ use aiken_lang::{ expr::UntypedExpr, - test_framework::{PropertyTestResult, TestResult, UnitTestResult}, + test_framework::{BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult}, }; pub use json::{json_schema, Json}; use std::{ @@ -10,6 +10,7 @@ use std::{ path::PathBuf, }; pub use terminal::Terminal; +use uplc::machine::cost_model::ExBudget; mod json; mod terminal; @@ -117,6 +118,18 @@ pub(crate) fn group_by_module( } pub(crate) fn find_max_execution_units(xs: &[TestResult]) -> (usize, usize, usize) { + fn max_execution_units(max_mem: i64, max_cpu: i64, cost: &ExBudget) -> (i64, i64) { + if cost.mem >= max_mem && cost.cpu >= max_cpu { + (cost.mem, cost.cpu) + } else if cost.mem > max_mem { + (cost.mem, max_cpu) + } else if cost.cpu > max_cpu { + (max_mem, cost.cpu) + } else { + (max_mem, max_cpu) + } + } + let (max_mem, max_cpu, max_iter) = xs.iter() .fold((0, 0, 0), |(max_mem, max_cpu, max_iter), test| match test { @@ -124,18 +137,15 @@ pub(crate) fn find_max_execution_units(xs: &[TestResult]) -> (usize, us (max_mem, max_cpu, std::cmp::max(max_iter, *iterations)) } TestResult::UnitTestResult(UnitTestResult { spent_budget, .. }) => { - if spent_budget.mem >= max_mem && spent_budget.cpu >= max_cpu { - (spent_budget.mem, spent_budget.cpu, max_iter) - } else if spent_budget.mem > max_mem { - (spent_budget.mem, max_cpu, max_iter) - } else if spent_budget.cpu > max_cpu { - (max_mem, spent_budget.cpu, max_iter) - } else { - (max_mem, max_cpu, max_iter) - } + let (max_mem, max_cpu) = max_execution_units(max_mem, max_cpu, spent_budget); + (max_mem, max_cpu, max_iter) } - TestResult::BenchmarkResult(..) => { - unreachable!("unexpected benchmark found amongst test results.") + TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => { + let (mut max_mem, mut max_cpu) = (max_mem, max_cpu); + for (_, measure) in measures { + (max_mem, max_cpu) = max_execution_units(max_mem, max_cpu, measure); + } + (max_mem, max_cpu, max_iter) } }); diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index 2f1adefd..813ab488 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -4,11 +4,21 @@ use aiken_lang::{ ast::OnTestFailure, expr::UntypedExpr, format::Formatter, - test_framework::{AssertionStyleOptions, PropertyTestResult, TestResult, UnitTestResult}, + test_framework::{ + AssertionStyleOptions, BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult, + }, }; use owo_colors::{OwoColorize, Stream::Stderr}; +use rgb::RGB8; +use std::sync::LazyLock; use uplc::machine::cost_model::ExBudget; +static BENCH_PLOT_COLOR: LazyLock = LazyLock::new(|| RGB8 { + r: 250, + g: 211, + b: 144, +}); + #[derive(Debug, Default, Clone, Copy)] pub struct Terminal; @@ -224,8 +234,45 @@ impl EventListener for Terminal { "...".if_supports_color(Stderr, |s| s.bold()) ); } - Event::FinishedBenchmarks { .. } => { - eprintln!("TODO: FinishedBenchmarks"); + Event::FinishedBenchmarks { seed, benchmarks } => { + let (max_mem, max_cpu, max_iter) = find_max_execution_units(&benchmarks); + + for (module, results) in &group_by_module(&benchmarks) { + let title = module + .if_supports_color(Stderr, |s| s.bold()) + .if_supports_color(Stderr, |s| s.blue()) + .to_string(); + + let benchmarks = results + .iter() + .map(|r| fmt_test(r, max_mem, max_cpu, max_iter, true)) + .collect::>() + .join("\n"); + + let seed_info = format!( + "with {opt}={seed}", + opt = "--seed".if_supports_color(Stderr, |s| s.bold()), + seed = format!("{seed}").if_supports_color(Stderr, |s| s.bold()) + ); + + if !benchmarks.is_empty() { + println!(); + } + + println!( + "{}\n", + pretty::indent( + &pretty::open_box(&title, &benchmarks, &seed_info, |border| border + .if_supports_color(Stderr, |s| s.bright_black()) + .to_string()), + 4 + ) + ); + } + + if !benchmarks.is_empty() { + println!(); + } } } } @@ -239,7 +286,9 @@ fn fmt_test( styled: bool, ) -> String { // Status - let mut test = if result.is_success() { + let mut test = if matches!(result, TestResult::BenchmarkResult { .. }) { + String::new() + } else if result.is_success() { pretty::style_if(styled, "PASS".to_string(), |s| { s.if_supports_color(Stderr, |s| s.bold()) .if_supports_color(Stderr, |s| s.green()) @@ -285,18 +334,73 @@ fn fmt_test( if *iterations > 1 { "s" } else { "" } ); } - TestResult::BenchmarkResult(..) => { - unreachable!("unexpected benchmark found amongst test results.") + TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => { + let max_size = measures + .iter() + .map(|(size, _)| *size) + .max() + .unwrap_or_default(); + + let mem_chart = format!( + "{title}\n{chart}", + title = "memory units" + .if_supports_color(Stderr, |s| s.yellow()) + .if_supports_color(Stderr, |s| s.bold()), + chart = plot( + &BENCH_PLOT_COLOR, + measures + .iter() + .map(|(size, budget)| (*size as f32, budget.mem as f32)) + .collect::>(), + max_size + ) + ); + + let cpu_chart = format!( + "{title}\n{chart}", + title = "cpu units" + .if_supports_color(Stderr, |s| s.yellow()) + .if_supports_color(Stderr, |s| s.bold()), + chart = plot( + &BENCH_PLOT_COLOR, + measures + .iter() + .map(|(size, budget)| (*size as f32, budget.cpu as f32)) + .collect::>(), + max_size + ) + ); + + let charts = mem_chart + .lines() + .zip(cpu_chart.lines()) + .map(|(l, r)| format!(" {}{r}", pretty::pad_right(l.to_string(), 55, " "))) + .collect::>() + .join("\n"); + + test = format!("{test}{charts}",); } } // Title - test = format!( - "{test} {title}", - title = pretty::style_if(styled, result.title().to_string(), |s| s - .if_supports_color(Stderr, |s| s.bright_blue()) - .to_string()) - ); + test = match result { + TestResult::BenchmarkResult(..) => { + format!( + "{title}\n{test}\n", + title = pretty::style_if(styled, result.title().to_string(), |s| s + .if_supports_color(Stderr, |s| s.bright_blue()) + .to_string()) + ) + } + TestResult::UnitTestResult(..) | TestResult::PropertyTestResult(..) => { + format!( + "{test} {title}", + title = pretty::style_if(styled, result.title().to_string(), |s| s + .if_supports_color(Stderr, |s| s.bright_blue()) + .to_string()) + ) + } + }; // Annotations match result { @@ -452,3 +556,14 @@ fn fmt_test_summary(tests: &[&TestResult], styled: bool) -> String { .to_string()), ) } + +fn plot(color: &RGB8, points: Vec<(f32, f32)>, max_size: usize) -> String { + use textplots::{Chart, ColorPlot, Shape}; + let mut chart = Chart::new(80, 50, 1.0, max_size as f32); + let plot = Shape::Lines(&points); + let chart = chart.linecolorplot(&plot, *color); + chart.borders(); + chart.axis(); + chart.figures(); + chart.to_string() +} From 8edd8d37dbed78e9efc6668dcb76ec17add6ab4a Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:17:15 +0100 Subject: [PATCH 12/15] fix benchmark output when either the sampler or bench fails This is likely even better than what was done for property testing. We shall revise that one perhaps one day. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 59 +++++++++++++++---- crates/aiken-project/src/error.rs | 8 ++- crates/aiken-project/src/lib.rs | 11 ++-- crates/aiken-project/src/options.rs | 2 +- .../aiken-project/src/telemetry/terminal.rs | 42 +++++++++---- crates/aiken/src/cmd/benchmark.rs | 40 ++++++++++++- 6 files changed, 131 insertions(+), 31 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 01887356..821d04f3 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -274,7 +274,7 @@ pub struct Fuzzer { } #[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)] -#[error("Fuzzer exited unexpectedly: {uplc_error}")] +#[error("Fuzzer exited unexpectedly: {uplc_error}.")] pub struct FuzzerError { traces: Vec, uplc_error: uplc::machine::Error, @@ -494,6 +494,29 @@ pub struct Sampler { pub stripped_type_info: Rc, } +#[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)] +pub enum BenchmarkError { + #[error("Sampler exited unexpectedly: {uplc_error}.")] + SamplerError { + traces: Vec, + uplc_error: uplc::machine::Error, + }, + #[error("Bench exited unexpectedly: {uplc_error}.")] + BenchError { + traces: Vec, + uplc_error: uplc::machine::Error, + }, +} + +impl BenchmarkError { + pub fn traces(&self) -> &[String] { + match self { + BenchmarkError::SamplerError { traces, .. } + | BenchmarkError::BenchError { traces, .. } => traces.as_slice(), + } + } +} + #[derive(Debug, Clone)] pub struct Benchmark { pub input_path: PathBuf, @@ -517,10 +540,10 @@ impl Benchmark { ) -> BenchmarkResult { let mut measures = Vec::with_capacity(max_size); let mut prng = Prng::from_seed(seed); - let mut success = true; + let mut error = None; let mut size = 0; - while success && max_size >= size { + while error.is_none() && max_size >= size { let fuzzer = self .sampler .program @@ -533,11 +556,24 @@ impl Benchmark { Ok(Some((new_prng, value))) => { prng = new_prng; - measures.push((size, self.eval(&value, plutus_version).cost())) + let mut result = self.eval(&value, plutus_version); + match result.result() { + Ok(_) => measures.push((size, result.cost())), + Err(uplc_error) => { + error = Some(BenchmarkError::BenchError { + traces: result + .logs() + .into_iter() + .filter(|s| PropertyTest::extract_label(s).is_none()) + .collect(), + uplc_error, + }); + } + } } - Err(_e) => { - success = false; + Err(FuzzerError { traces, uplc_error }) => { + error = Some(BenchmarkError::SamplerError { traces, uplc_error }); } } @@ -547,7 +583,7 @@ impl Benchmark { BenchmarkResult { bench: self, measures, - success, + error, } } @@ -650,7 +686,6 @@ impl Prng { pub fn sample( &self, fuzzer: &Program, - // iteration: usize, ) -> Result, FuzzerError> { let program = Program::::try_from(fuzzer.apply_data(self.uplc())).unwrap(); let mut result = program.eval(ExBudget::max()); @@ -1107,7 +1142,7 @@ impl TestResult { } OnTestFailure::SucceedImmediately => counterexample.is_some(), }, - TestResult::BenchmarkResult(BenchmarkResult { success, .. }) => *success, + TestResult::BenchmarkResult(BenchmarkResult { error, .. }) => error.is_none(), } } @@ -1135,7 +1170,9 @@ impl TestResult { match self { TestResult::UnitTestResult(UnitTestResult { traces, .. }) | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) => traces, - TestResult::BenchmarkResult(BenchmarkResult { .. }) => &[], + TestResult::BenchmarkResult(BenchmarkResult { error, .. }) => { + error.as_ref().map(|e| e.traces()).unwrap_or_default() + } } } } @@ -1475,7 +1512,7 @@ impl Assertion { pub struct BenchmarkResult { pub bench: Benchmark, pub measures: Vec<(usize, ExBudget)>, - pub success: bool, + pub error: Option, } unsafe impl Send for BenchmarkResult {} diff --git a/crates/aiken-project/src/error.rs b/crates/aiken-project/src/error.rs index 656e19f8..0c210121 100644 --- a/crates/aiken-project/src/error.rs +++ b/crates/aiken-project/src/error.rs @@ -3,7 +3,7 @@ use aiken_lang::{ ast::{self, Span}, error::ExtraData, parser::error::ParseError, - test_framework::{PropertyTestResult, TestResult, UnitTestResult}, + test_framework::{BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult}, tipo, }; use miette::{ @@ -193,7 +193,11 @@ impl Error { test.input_path.to_path_buf(), test.program.to_pretty(), ), - TestResult::BenchmarkResult(_) => ("bench".to_string(), PathBuf::new(), String::new()), // todo + TestResult::BenchmarkResult(BenchmarkResult { bench, .. }) => ( + bench.name.to_string(), + bench.input_path.to_path_buf(), + bench.program.to_pretty(), + ), }; Error::TestFailure { diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index 9e343be5..7b1cc13f 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -302,17 +302,18 @@ where match_benchmarks: Option>, exact_match: bool, seed: u32, - iterations: usize, + max_size: usize, + tracing: Tracing, env: Option, ) -> Result<(), Vec> { let options = Options { - tracing: Tracing::silent(), + tracing, env, code_gen_mode: CodeGenMode::Benchmark { match_benchmarks, exact_match, seed, - iterations, + max_size, }, blueprint_path: self.blueprint_path(None), }; @@ -469,7 +470,7 @@ where match_benchmarks, exact_match, seed, - iterations, + max_size, } => { let verbose = false; @@ -484,7 +485,7 @@ where self.event_listener.handle_event(Event::RunningBenchmarks); } - let benchmarks = self.run_runnables(benchmarks, seed, iterations); + let benchmarks = self.run_runnables(benchmarks, seed, max_size); let errors: Vec = benchmarks .iter() diff --git a/crates/aiken-project/src/options.rs b/crates/aiken-project/src/options.rs index 8b4fdf9b..0e5706d6 100644 --- a/crates/aiken-project/src/options.rs +++ b/crates/aiken-project/src/options.rs @@ -33,7 +33,7 @@ pub enum CodeGenMode { match_benchmarks: Option>, exact_match: bool, seed: u32, - iterations: usize, + max_size: usize, }, NoOp, } diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index 813ab488..afacfcef 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -247,7 +247,10 @@ impl EventListener for Terminal { .iter() .map(|r| fmt_test(r, max_mem, max_cpu, max_iter, true)) .collect::>() - .join("\n"); + .join("\n") + .chars() + .skip(1) // Remove extra first newline + .collect::(); let seed_info = format!( "with {opt}={seed}", @@ -287,7 +290,21 @@ fn fmt_test( ) -> String { // Status let mut test = if matches!(result, TestResult::BenchmarkResult { .. }) { - String::new() + format!( + "\n{label}{title}\n", + label = if result.is_success() { + String::new() + } else { + pretty::style_if(styled, "FAIL ".to_string(), |s| { + s.if_supports_color(Stderr, |s| s.bold()) + .if_supports_color(Stderr, |s| s.red()) + .to_string() + }) + }, + title = pretty::style_if(styled, result.title().to_string(), |s| s + .if_supports_color(Stderr, |s| s.bright_blue()) + .to_string()) + ) } else if result.is_success() { pretty::style_if(styled, "PASS".to_string(), |s| { s.if_supports_color(Stderr, |s| s.bold()) @@ -334,7 +351,17 @@ fn fmt_test( if *iterations > 1 { "s" } else { "" } ); } - TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => { + TestResult::BenchmarkResult(BenchmarkResult { error: Some(e), .. }) => { + test = format!( + "{test}{}", + e.to_string().if_supports_color(Stderr, |s| s.red()) + ); + } + TestResult::BenchmarkResult(BenchmarkResult { + measures, + error: None, + .. + }) => { let max_size = measures .iter() .map(|(size, _)| *size) @@ -384,14 +411,7 @@ fn fmt_test( // Title test = match result { - TestResult::BenchmarkResult(..) => { - format!( - "{title}\n{test}\n", - title = pretty::style_if(styled, result.title().to_string(), |s| s - .if_supports_color(Stderr, |s| s.bright_blue()) - .to_string()) - ) - } + TestResult::BenchmarkResult(..) => test, TestResult::UnitTestResult(..) | TestResult::PropertyTestResult(..) => { format!( "{test} {title}", diff --git a/crates/aiken/src/cmd/benchmark.rs b/crates/aiken/src/cmd/benchmark.rs index 1a495f8d..11121f60 100644 --- a/crates/aiken/src/cmd/benchmark.rs +++ b/crates/aiken/src/cmd/benchmark.rs @@ -1,4 +1,8 @@ -use aiken_lang::test_framework::Benchmark; +use super::build::{trace_filter_parser, trace_level_parser}; +use aiken_lang::{ + ast::{TraceLevel, Tracing}, + test_framework::Benchmark, +}; use aiken_project::watch::with_project; use rand::prelude::*; use std::{ @@ -36,6 +40,34 @@ pub struct Args { /// Environment to use for benchmarking env: Option, + + /// Filter traces to be included in the generated program(s). + /// + /// - user-defined: + /// only consider traces that you've explicitly defined + /// either through the 'trace' keyword of via the trace-if-false + /// ('?') operator. + /// + /// - compiler-generated: + /// only included internal traces generated by the + /// Aiken compiler, for example in usage of 'expect'. + /// + /// - all: + /// include both user-defined and compiler-generated traces. + /// + /// [default: all] + #[clap(short = 'f', long, value_parser=trace_filter_parser(), default_missing_value="all", verbatim_doc_comment, alias="filter_traces")] + trace_filter: Option Tracing>, + + /// Choose the verbosity level of traces: + /// + /// - silent: disable traces altogether + /// - compact: only culprit line numbers are shown on failures + /// - verbose: enable full verbose traces as provided by the user or the compiler + /// + /// [optional] + #[clap(short, long, value_parser=trace_level_parser(), default_value_t=TraceLevel::Silent, verbatim_doc_comment)] + trace_level: TraceLevel, } pub fn exec( @@ -46,6 +78,8 @@ pub fn exec( seed, max_size, env, + trace_filter, + trace_level, }: Args, ) -> miette::Result<()> { let mut rng = rand::thread_rng(); @@ -62,6 +96,10 @@ pub fn exec( exact_match, seed, max_size, + match trace_filter { + Some(trace_filter) => trace_filter(trace_level), + None => Tracing::All(trace_level), + }, env.clone(), ) }, From 451179fd49c6c1e76eee5d7bbf5928324fc5a269 Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:28:42 +0100 Subject: [PATCH 13/15] Update CHANGELOG w.r.t benchmarks Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d64386d8..17f5063a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,21 @@ ## v1.1.11 - UNRELEASED +### Added + +- **aiken**: New `aiken bench` command to run benchmarks. @Riley-Kilgore, @KtorZ + + The command is very similar to `aiken check`, and will collect and run benchmarks found across the codebase. The output by default is a set of pretty terminal plots for each dimension (mem & cpu) for each test bench. The complete dataset of points can be obtained in a structured (JSON) format by redirecting the output to a file. + +- **aiken-lang**: New `bench` keyword and capabilities to the test framework. @Riley-Kilgore, @KtorZ + + A `bench` is a new type of test that takes in a single `Sampler = fn(Int) -> Fuzzer` as parameter, similar to how property-based test receive `Fuzzer`. A `Sampler` is in fact, a _scaled Fuzzer_ which receive a monotically increasing size as parameter. This allows fine-grained control over generated values. Unlike tests, benchmarks can return _anything_ since their output is ignored. + + Read more about benchmarks in the [user manual](https://aiken-lang.org/language-tour/bench). + ### Changed -- **aiken**: support for `bench` keyword to define benchmarks. @Riley-Kilgore + - **aiken-lang**: The compiler now raises a warning when attempting to destructure a record constructor without using named fields. See [#1084](https://github.com/aiken-lang/aiken/issues/1084). @KtorZ - **aiken-lang**: Fix blueprint schema definitions related to pairs (no longer omit (sometimes) Pairs definitions, and generate them as data List). See [#1086](https://github.com/aiken-lang/aiken/issues/1086) and [#970](https://github.com/aiken-lang/aiken/issues/970). @KtorZ From d53f770d9006519b2e4389345331d61597c08a7c Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:53:58 +0100 Subject: [PATCH 14/15] minor tweaks and proof-reading. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 2 +- crates/aiken-project/Cargo.toml | 2 +- crates/aiken/src/cmd/benchmark.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 821d04f3..57ba0491 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -530,7 +530,7 @@ pub struct Benchmark { unsafe impl Send for Benchmark {} impl Benchmark { - pub const DEFAULT_MAX_SIZE: usize = 10; + pub const DEFAULT_MAX_SIZE: usize = 30; pub fn run( self, diff --git a/crates/aiken-project/Cargo.toml b/crates/aiken-project/Cargo.toml index 30cccd28..bb2c168f 100644 --- a/crates/aiken-project/Cargo.toml +++ b/crates/aiken-project/Cargo.toml @@ -11,7 +11,7 @@ authors = [ "Kasey White ", "KtorZ ", ] -rust-version = "1.70.0" +rust-version = "1.80.0" build = "build.rs" [dependencies] diff --git a/crates/aiken/src/cmd/benchmark.rs b/crates/aiken/src/cmd/benchmark.rs index 11121f60..3b09a928 100644 --- a/crates/aiken/src/cmd/benchmark.rs +++ b/crates/aiken/src/cmd/benchmark.rs @@ -17,7 +17,7 @@ pub struct Args { /// Path to project directory: Option, - /// An initial seed to initialize the pseudo-random generator for property-tests. + /// An initial seed to initialize the pseudo-random generator for benchmarks. #[clap(long)] seed: Option, @@ -29,7 +29,7 @@ pub struct Args { /// Only run benchmarks if they match any of these strings. /// /// You can match a module with `-m aiken/list` or `-m list`. - /// You can match a test with `-m "aiken/list.{map}"` or `-m "aiken/option.{flatten_1}"` + /// You can match a benchmark with `-m "aiken/list.{map}"` or `-m "aiken/option.{flatten_1}"` #[clap(short, long)] match_benchmarks: Option>, From a6cdb5583d3244a80c9f822f685eb771e97f345b Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:57:43 +0100 Subject: [PATCH 15/15] Make nix build optional. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- .github/workflows/nix.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index f3e6aacd..0b87339f 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -6,7 +6,6 @@ on: jobs: nix-build: runs-on: ubuntu-latest - steps: - name: Checkout repository uses: actions/checkout@v3 @@ -20,5 +19,14 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@v1 - name: Build Aiken - run: nix build - + shell: bash + run: | + set +e + nix build + exitcode="$?" + if [[ "$exitcode" != "0" ]] ; then + echo "::warning::Nix build failed with exit code $exitcode" + exit 0 + else + exit "$exitcode" + fi