diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index 846162e0..4287d830 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -297,6 +297,26 @@ where self.compile(options) } + pub fn benchmark( + &mut self, + match_tests: Option>, + exact_match: bool, + seed: u32, + property_max_success: usize, + ) -> Result<(), Vec> { + let options = Options { + tracing: Tracing::silent(), + code_gen_mode: CodeGenMode::Benchmark { + match_tests, + exact_match, + seed, + property_max_success, + } + }; + + self.compile(options) + } + pub fn dump_uplc(&self, blueprint: &Blueprint) -> Result<(), Error> { let dir = self.root.join("artifacts"); @@ -403,7 +423,7 @@ where property_max_success, } => { let tests = - self.collect_tests(verbose, match_tests, exact_match, options.tracing)?; + self.collect_tests(false, match_tests, exact_match, options.tracing)?; if !tests.is_empty() { self.event_listener.handle_event(Event::RunningTests); @@ -442,6 +462,52 @@ where Ok(()) } } + CodeGenMode::Benchmark { + match_tests, + exact_match, + seed, + property_max_success, + } => { + let tests = + self.collect_tests(false, match_tests, exact_match, options.tracing)?; + + if !tests.is_empty() { + self.event_listener.handle_event(Event::RunningTests); + } + + let tests = self.run_benchmarks(tests, seed, property_max_success); + + self.checks_count = if tests.is_empty() { + None + } else { + Some(tests.iter().fold(0, |acc, test| { + acc + match test { + TestResult::PropertyTestResult(r) => r.iterations, + _ => 1, + } + })) + }; + + let errors: Vec = tests + .iter() + .filter_map(|e| { + if e.is_success() { + None + } else { + Some(e.into_error(false)) + } + }) + .collect(); + + self.event_listener + .handle_event(Event::FinishedTests { seed, tests }); + + if !errors.is_empty() { + Err(errors) + } else { + Ok(()) + } + } CodeGenMode::NoOp => Ok(()), } } @@ -1020,6 +1086,32 @@ where .collect() } + fn run_benchmarks( + &self, + tests: Vec, + seed: u32, + property_max_success: usize, + ) -> Vec> { + use rayon::prelude::*; + + let data_types = utils::indexmap::as_ref_values(&self.data_types); + + let plutus_version = &self.config.plutus; + + tests + .into_par_iter() + .map(|test| match test { + Test::UnitTest(unit_test) => unit_test.run(plutus_version), + Test::PropertyTest(property_test) => { + property_test.run(seed, property_max_success, plutus_version) + } + }) + .collect::), PlutusData>>>() + .into_iter() + .map(|test| test.reify(&data_types)) + .collect() + } + fn aiken_files(&mut self, dir: &Path, kind: ModuleKind) -> Result<(), Error> { let mut has_default = None; diff --git a/crates/aiken-project/src/options.rs b/crates/aiken-project/src/options.rs index 8e6a3019..6e87ec52 100644 --- a/crates/aiken-project/src/options.rs +++ b/crates/aiken-project/src/options.rs @@ -28,5 +28,12 @@ pub enum CodeGenMode { property_max_success: usize, }, Build(bool), + Benchmark { + match_tests: Option>, + exact_match: bool, + seed: u32, + property_max_success: usize + + }, NoOp, } diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index fb59c5d0..87ab30d0 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -43,10 +43,15 @@ pub enum Event { path: PathBuf, }, RunningTests, + RunningBenchmarks, FinishedTests { seed: u32, tests: Vec>, }, + FinishedBenchmarks { + seed: u32, + tests: Vec>, + }, WaitingForBuildDirLock, ResolvingPackages { name: String, diff --git a/crates/aiken/src/cmd/benchmark.rs b/crates/aiken/src/cmd/benchmark.rs new file mode 100644 index 00000000..cdeef3df --- /dev/null +++ b/crates/aiken/src/cmd/benchmark.rs @@ -0,0 +1,61 @@ +use super::build::{filter_traces_parser, trace_level_parser}; +use aiken_lang::ast::{TraceLevel, Tracing}; +use aiken_project::{ + test_framework::PropertyTest, + watch::{self, watch_project, with_project}, +}; +use rand::prelude::*; +use std::{path::PathBuf, process}; + +#[derive(clap::Args)] +/// Type-check an Aiken project +pub struct Args { + /// Path to project + directory: Option, + + /// An initial seed to initialize the pseudo-random generator for property-tests. + #[clap(long)] + seed: Option, + + /// Maximum number of successful test run for considering a property-based test valid. + #[clap(long, default_value_t = PropertyTest::DEFAULT_MAX_SUCCESS)] + max_success: usize, + + /// Only run tests if they match any of these strings. + /// You can match a module with `-m aiken/list` or `-m list`. + /// You can match a test with `-m "aiken/list.{map}"` or `-m "aiken/option.{flatten_1}"` + #[clap(short, long)] + match_tests: Option>, + + /// This is meant to be used with `--match-tests`. + /// It forces test names to match exactly + #[clap(short, long)] + exact_match: bool, +} + +pub fn exec( + Args { + directory, + match_tests, + exact_match, + seed, + max_success, + }: Args, +) -> miette::Result<()> { + // Actually we don't want to use check right? + let mut rng = rand::thread_rng(); + + let seed = seed.unwrap_or_else(|| rng.gen()); + + let result = with_project(directory.as_deref(), false, |p| { + // We don't want to check here, we want to benchmark + p.benchmark( + match_tests.clone(), + exact_match, + seed, + max_success + ) + }); +// todo riley - We need to either print or output the results to a file. + result.map_err(|_| process::exit(1)) +} \ No newline at end of file diff --git a/crates/aiken/src/cmd/mod.rs b/crates/aiken/src/cmd/mod.rs index 057c676e..a4675819 100644 --- a/crates/aiken/src/cmd/mod.rs +++ b/crates/aiken/src/cmd/mod.rs @@ -1,6 +1,7 @@ use aiken_project::config; use clap::Parser; +pub mod benchmark; pub mod blueprint; pub mod build; pub mod check; @@ -35,6 +36,8 @@ pub enum Cmd { Docs(docs::Args), Add(packages::add::Args), + Benchmark(benchmark::Args), + #[clap(subcommand)] Blueprint(blueprint::Cmd), diff --git a/crates/aiken/src/main.rs b/crates/aiken/src/main.rs index cd23a9e4..8e1fce35 100644 --- a/crates/aiken/src/main.rs +++ b/crates/aiken/src/main.rs @@ -2,6 +2,7 @@ use aiken_project::{config, pretty}; #[cfg(not(target_os = "windows"))] use cmd::completion; use cmd::{ + benchmark, blueprint::{self, address}, build, check, docs, export, fmt, lsp, new, packages::{self, add}, @@ -23,6 +24,7 @@ fn main() -> miette::Result<()> { Cmd::Build(args) => build::exec(args), Cmd::Address(args) => address::exec(args), Cmd::Check(args) => check::exec(args), + Cmd::Benchmark(args) => benchmark::exec(args), Cmd::Docs(args) => docs::exec(args), Cmd::Add(args) => add::exec(args), Cmd::Blueprint(args) => blueprint::exec(args),