From 2dbc33e91f12e97d857b9954e6ebbf7cd956874f Mon Sep 17 00:00:00 2001 From: KtorZ <5680256+KtorZ@users.noreply.github.com> Date: Sat, 8 Feb 2025 17:40:07 +0100 Subject: [PATCH] fuse together bench & test runners, and collect all bench measures. This commit removes some duplication between bench and test runners, as well as fixing the results coming out of running benchmarks. Running benchmarks is expected to yield multiple measures, for each of the iteration. For now, it'll suffice to show results for each size; but eventually, we'll possibly try to interpolate results with different curves and pick the best candidate. Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com> --- crates/aiken-lang/src/test_framework.rs | 123 +++++++++--------- crates/aiken-project/src/lib.rs | 120 ++++++----------- crates/aiken-project/src/options.rs | 4 +- crates/aiken-project/src/telemetry.rs | 2 +- crates/aiken-project/src/telemetry/json.rs | 9 +- .../aiken-project/src/telemetry/terminal.rs | 26 +--- crates/aiken-project/src/test_framework.rs | 41 +++--- 7 files changed, 134 insertions(+), 191 deletions(-) diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 85131125..cf0b66e3 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -28,6 +28,12 @@ use uplc::{ }; use vec1::{vec1, Vec1}; +#[derive(Debug, Clone, Copy)] +pub enum RunnableKind { + Test, + Bench, +} + /// ----- Test ----------------------------------------------------------------- /// /// Aiken supports two kinds of tests: unit and property. A unit test is a simply @@ -117,15 +123,15 @@ impl Test { }) } - pub fn from_test_definition( + pub fn from_function_definition( generator: &mut CodeGenerator<'_>, test: TypedTest, module_name: String, input_path: PathBuf, - is_benchmark: bool, + kind: RunnableKind, ) -> Test { if test.arguments.is_empty() { - if is_benchmark { + if matches!(kind, RunnableKind::Bench) { unreachable!("benchmark must have at least one argument"); } else { Self::unit_test(generator, test, module_name, input_path) @@ -153,8 +159,8 @@ impl Test { // apply onto it later. let generator_program = generator.clone().generate_raw(&via, &[], &module_name); - if is_benchmark { - Test::Benchmark(Benchmark { + match kind { + RunnableKind::Bench => Test::Benchmark(Benchmark { input_path, module: module_name, name: test.name, @@ -165,9 +171,8 @@ impl Test { type_info, stripped_type_info, }, - }) - } else { - Self::property_test( + }), + RunnableKind::Test => Self::property_test( input_path, module_name, test.name, @@ -178,27 +183,26 @@ impl Test { stripped_type_info, type_info, }, - ) + ), } } } - pub fn from_benchmark_definition( - generator: &mut CodeGenerator<'_>, - test: TypedTest, - module_name: String, - input_path: PathBuf, - ) -> Test { - Self::from_test_definition(generator, test, module_name, input_path, true) - } - - pub fn from_function_definition( - generator: &mut CodeGenerator<'_>, - test: TypedTest, - module_name: String, - input_path: PathBuf, - ) -> Test { - Self::from_test_definition(generator, test, module_name, input_path, false) + pub fn run( + self, + seed: u32, + max_success: usize, + plutus_version: &PlutusVersion, + ) -> TestResult<(Constant, Rc), PlutusData> { + match self { + Test::UnitTest(unit_test) => TestResult::UnitTestResult(unit_test.run(plutus_version)), + Test::PropertyTest(property_test) => { + TestResult::PropertyTestResult(property_test.run(seed, max_success, plutus_version)) + } + Test::Benchmark(benchmark) => { + TestResult::BenchmarkResult(benchmark.run(seed, max_success, plutus_version)) + } + } } } @@ -217,7 +221,7 @@ pub struct UnitTest { unsafe impl Send for UnitTest {} impl UnitTest { - pub fn run(self, plutus_version: &PlutusVersion) -> TestResult<(Constant, Rc), T> { + pub fn run(self, plutus_version: &PlutusVersion) -> UnitTestResult<(Constant, Rc)> { let mut eval_result = Program::::try_from(self.program.clone()) .unwrap() .eval_version(ExBudget::max(), &plutus_version.into()); @@ -233,13 +237,13 @@ impl UnitTest { } traces.extend(eval_result.logs()); - TestResult::UnitTestResult(UnitTestResult { + UnitTestResult { success, test: self.to_owned(), spent_budget: eval_result.cost(), traces, assertion: self.assertion, - }) + } } } @@ -317,12 +321,12 @@ impl PropertyTest { /// Run a property test from a given seed. The property is run at most DEFAULT_MAX_SUCCESS times. It /// may stops earlier on failure; in which case a 'counterexample' is returned. - pub fn run( + pub fn run( self, seed: u32, n: usize, plutus_version: &PlutusVersion, - ) -> TestResult { + ) -> PropertyTestResult { let mut labels = BTreeMap::new(); let mut remaining = n; @@ -352,13 +356,13 @@ impl PropertyTest { ), }; - TestResult::PropertyTestResult(PropertyTestResult { + PropertyTestResult { test: self, counterexample, iterations, labels, traces, - }) + } } pub fn run_n_times<'a>( @@ -503,51 +507,43 @@ pub struct Benchmark { unsafe impl Send for Benchmark {} impl Benchmark { - pub fn benchmark( + pub fn run( self, seed: u32, max_iterations: usize, plutus_version: &PlutusVersion, - ) -> Vec { - let mut results = Vec::with_capacity(max_iterations); + ) -> BenchmarkResult { + let mut measures = Vec::with_capacity(max_iterations); let mut iteration = 0; let mut prng = Prng::from_seed(seed); + let mut success = true; + + while success && max_iterations > iteration { + let size = Data::integer(num_bigint::BigInt::from(iteration as i64)); + let fuzzer = self.sampler.program.apply_data(size); - while max_iterations > iteration { - let fuzzer = self - .sampler - .program - .apply_data(Data::integer(num_bigint::BigInt::from(iteration as i64))); match prng.sample(&fuzzer) { - Ok(Some((new_prng, value))) => { - prng = new_prng; - let mut eval_result = self.eval(&value, plutus_version); - results.push(BenchmarkResult { - bench: self.clone(), - cost: eval_result.cost(), - success: true, - traces: eval_result.logs().to_vec(), - }); - } - Ok(None) => { panic!("A seeded PRNG returned 'None' which indicates a sampler is ill-formed and implemented wrongly; please contact library's authors."); } - Err(e) => { - results.push(BenchmarkResult { - bench: self.clone(), - cost: ExBudget::default(), - success: false, - traces: vec![e.to_string()], - }); - break; + Ok(Some((new_prng, value))) => { + prng = new_prng; + measures.push(self.eval(&value, plutus_version).cost()) + } + + Err(_e) => { + success = false; } } iteration += 1; } - results + BenchmarkResult { + bench: self, + measures, + success, + } } pub fn eval(&self, value: &PlutusData, plutus_version: &PlutusVersion) -> EvalResult { @@ -1133,8 +1129,8 @@ impl TestResult { pub fn traces(&self) -> &[String] { match self { TestResult::UnitTestResult(UnitTestResult { traces, .. }) - | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) - | TestResult::BenchmarkResult(BenchmarkResult { traces, .. }) => traces, + | TestResult::PropertyTestResult(PropertyTestResult { traces, .. }) => traces, + TestResult::BenchmarkResult(BenchmarkResult { .. }) => &[], } } } @@ -1473,9 +1469,8 @@ impl Assertion { #[derive(Debug, Clone)] pub struct BenchmarkResult { pub bench: Benchmark, - pub cost: ExBudget, + pub measures: Vec, pub success: bool, - pub traces: Vec, } unsafe impl Send for BenchmarkResult {} diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index 0f8e7c6c..9e343be5 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -40,7 +40,7 @@ use aiken_lang::{ format::{Formatter, MAX_COLUMNS}, gen_uplc::CodeGenerator, line_numbers::LineNumbers, - test_framework::{Test, TestResult}, + test_framework::{RunnableKind, Test, TestResult}, tipo::{Type, TypeInfo}, utils, IdGenerator, }; @@ -83,12 +83,6 @@ enum AddModuleBy { Path(PathBuf), } -#[derive(Debug, Clone, Copy)] -enum Runnable { - Test, - Bench, -} - pub struct Project where T: EventListener, @@ -305,20 +299,20 @@ where pub fn benchmark( &mut self, - match_tests: Option>, + match_benchmarks: Option>, exact_match: bool, seed: u32, - times_to_run: usize, + iterations: usize, env: Option, ) -> Result<(), Vec> { let options = Options { tracing: Tracing::silent(), env, code_gen_mode: CodeGenMode::Benchmark { - match_tests, + match_benchmarks, exact_match, seed, - times_to_run, + iterations, }, blueprint_path: self.blueprint_path(None), }; @@ -438,7 +432,7 @@ where self.event_listener.handle_event(Event::RunningTests); } - let tests = self.run_tests(tests, seed, property_max_success); + let tests = self.run_runnables(tests, seed, property_max_success); self.checks_count = if tests.is_empty() { None @@ -472,21 +466,25 @@ where } } CodeGenMode::Benchmark { - match_tests, + match_benchmarks, exact_match, seed, - times_to_run, + iterations, } => { let verbose = false; - let tests = - self.collect_benchmarks(verbose, match_tests, exact_match, options.tracing)?; + let benchmarks = self.collect_benchmarks( + verbose, + match_benchmarks, + exact_match, + options.tracing, + )?; - if !tests.is_empty() { + if !benchmarks.is_empty() { self.event_listener.handle_event(Event::RunningBenchmarks); } - let benchmarks = self.run_benchmarks(tests, seed, times_to_run); + let benchmarks = self.run_runnables(benchmarks, seed, iterations); let errors: Vec = benchmarks .iter() @@ -962,7 +960,7 @@ where fn collect_test_items( &mut self, - kind: Runnable, + kind: RunnableKind, verbose: bool, match_tests: Option>, exact_match: bool, @@ -1001,8 +999,8 @@ where for def in checked_module.ast.definitions() { let func = match (kind, def) { - (Runnable::Test, Definition::Test(func)) => Some(func), - (Runnable::Bench, Definition::Benchmark(func)) => Some(func), + (RunnableKind::Test, Definition::Test(func)) => Some(func), + (RunnableKind::Bench, Definition::Benchmark(func)) => Some(func), _ => None, }; @@ -1056,20 +1054,13 @@ where }) } - tests.push(match kind { - Runnable::Test => Test::from_function_definition( - &mut generator, - test.to_owned(), - module_name, - input_path, - ), - Runnable::Bench => Test::from_benchmark_definition( - &mut generator, - test.to_owned(), - module_name, - input_path, - ), - }); + tests.push(Test::from_function_definition( + &mut generator, + test.to_owned(), + module_name, + input_path, + kind, + )); } Ok(tests) @@ -1082,7 +1073,13 @@ where exact_match: bool, tracing: Tracing, ) -> Result, Error> { - self.collect_test_items(Runnable::Test, verbose, match_tests, exact_match, tracing) + self.collect_test_items( + RunnableKind::Test, + verbose, + match_tests, + exact_match, + tracing, + ) } fn collect_benchmarks( @@ -1092,14 +1089,20 @@ where exact_match: bool, tracing: Tracing, ) -> Result, Error> { - self.collect_test_items(Runnable::Bench, verbose, match_tests, exact_match, tracing) + self.collect_test_items( + RunnableKind::Bench, + verbose, + match_tests, + exact_match, + tracing, + ) } - fn run_tests( + fn run_runnables( &self, tests: Vec, seed: u32, - property_max_success: usize, + max_success: usize, ) -> Vec> { use rayon::prelude::*; @@ -1109,44 +1112,7 @@ where tests .into_par_iter() - .map(|test| match test { - Test::UnitTest(unit_test) => unit_test.run(plutus_version), - Test::PropertyTest(property_test) => { - property_test.run(seed, property_max_success, plutus_version) - } - Test::Benchmark(_) => { - unreachable!("found unexpected benchmark amongst collected tests.") - } - }) - .collect::), PlutusData>>>() - .into_iter() - .map(|test| test.reify(&data_types)) - .collect() - } - - fn run_benchmarks( - &self, - tests: Vec, - seed: u32, - property_max_success: usize, - ) -> Vec> { - use rayon::prelude::*; - - let data_types = utils::indexmap::as_ref_values(&self.data_types); - let plutus_version = &self.config.plutus; - - tests - .into_par_iter() - .flat_map(|test| match test { - Test::UnitTest(_) | Test::PropertyTest(_) => { - unreachable!("Tests cannot be ran during benchmarking.") - } - Test::Benchmark(benchmark) => benchmark - .benchmark(seed, property_max_success, plutus_version) - .into_iter() - .map(TestResult::BenchmarkResult) - .collect::>(), - }) + .map(|test| test.run(seed, max_success, plutus_version)) .collect::), PlutusData>>>() .into_iter() .map(|test| test.reify(&data_types)) diff --git a/crates/aiken-project/src/options.rs b/crates/aiken-project/src/options.rs index 2afa69ed..8b4fdf9b 100644 --- a/crates/aiken-project/src/options.rs +++ b/crates/aiken-project/src/options.rs @@ -30,10 +30,10 @@ pub enum CodeGenMode { }, Build(bool), Benchmark { - match_tests: Option>, + match_benchmarks: Option>, exact_match: bool, seed: u32, - times_to_run: usize, + iterations: usize, }, NoOp, } diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index 0e3e7d19..5ef77d13 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -135,7 +135,7 @@ pub(crate) fn find_max_execution_units(xs: &[TestResult]) -> (usize, us } } TestResult::BenchmarkResult(..) => { - unreachable!("property returned benchmark result ?!") + unreachable!("unexpected benchmark found amongst test results.") } }); diff --git a/crates/aiken-project/src/telemetry/json.rs b/crates/aiken-project/src/telemetry/json.rs index ee76b0f5..e9c37623 100644 --- a/crates/aiken-project/src/telemetry/json.rs +++ b/crates/aiken-project/src/telemetry/json.rs @@ -47,8 +47,13 @@ impl EventListener for Json { Some(serde_json::json!({ "name": result.bench.name, "module": result.bench.module, - "memory": result.cost.mem, - "cpu": result.cost.cpu + "measures": result.measures + .into_iter() + .map(|measure| serde_json::json!({ + "memory": measure.mem, + "cpu": measure.cpu + })) + .collect::>() })) } else { None diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index 16e2578e..2f1adefd 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -224,15 +224,8 @@ impl EventListener for Terminal { "...".if_supports_color(Stderr, |s| s.bold()) ); } - Event::FinishedBenchmarks { benchmarks, .. } => { - for bench in benchmarks { - if let TestResult::BenchmarkResult(result) = bench { - println!("{} {} ", result.bench.name.bold(), "BENCH".blue(),); - println!(" Memory: {} bytes", result.cost.mem); - println!(" CPU: {} units", result.cost.cpu); - println!(); - } - } + Event::FinishedBenchmarks { .. } => { + eprintln!("TODO: FinishedBenchmarks"); } } } @@ -292,19 +285,8 @@ fn fmt_test( if *iterations > 1 { "s" } else { "" } ); } - TestResult::BenchmarkResult(benchmark) => { - let mem_pad = pretty::pad_left(benchmark.cost.mem.to_string(), max_mem, " "); - let cpu_pad = pretty::pad_left(benchmark.cost.cpu.to_string(), max_cpu, " "); - - test = format!( - "{test} [mem: {mem_unit}, cpu: {cpu_unit}]", - mem_unit = pretty::style_if(styled, mem_pad, |s| s - .if_supports_color(Stderr, |s| s.cyan()) - .to_string()), - cpu_unit = pretty::style_if(styled, cpu_pad, |s| s - .if_supports_color(Stderr, |s| s.cyan()) - .to_string()), - ); + TestResult::BenchmarkResult(..) => { + unreachable!("unexpected benchmark found amongst test results.") } } diff --git a/crates/aiken-project/src/test_framework.rs b/crates/aiken-project/src/test_framework.rs index f8e2dc7f..1cd509a1 100644 --- a/crates/aiken-project/src/test_framework.rs +++ b/crates/aiken-project/src/test_framework.rs @@ -101,6 +101,7 @@ mod test { test.to_owned(), module_name.to_string(), PathBuf::new(), + RunnableKind::Test, ), data_types, ) @@ -245,13 +246,12 @@ mod test { } "#}); - assert!(prop - .run::<()>( - 42, - PropertyTest::DEFAULT_MAX_SUCCESS, - &PlutusVersion::default() - ) - .is_success()); + assert!(TestResult::PropertyTestResult::<(), _>(prop.run( + 42, + PropertyTest::DEFAULT_MAX_SUCCESS, + &PlutusVersion::default() + )) + .is_success()); } #[test] @@ -273,25 +273,20 @@ mod test { } "#}); - match prop.run::<()>( + let result = prop.run( 42, PropertyTest::DEFAULT_MAX_SUCCESS, &PlutusVersion::default(), - ) { - TestResult::BenchmarkResult(..) | TestResult::UnitTestResult(..) => { - unreachable!("property returned non-property result ?!") - } - TestResult::PropertyTestResult(result) => { - assert!( - result - .labels - .iter() - .eq(vec![(&"head".to_string(), &53), (&"tail".to_string(), &47)]), - "labels: {:#?}", - result.labels - ) - } - } + ); + + assert!( + result + .labels + .iter() + .eq(vec![(&"head".to_string(), &53), (&"tail".to_string(), &47)]), + "labels: {:#?}", + result.labels + ); } #[test]