Dump (benchmarking wip)

crates/aiken-project/src/lib.rs

@@ -297,6 +297,26 @@ where
         self.compile(options)
     }
 
+    pub fn benchmark(
+        &mut self,
+        match_tests: Option<Vec<String>>,
+        exact_match: bool,
+        seed: u32,
+        property_max_success: usize,
+    ) -> Result<(), Vec<Error>> {
+        let options = Options {
+            tracing: Tracing::silent(),
+            code_gen_mode: CodeGenMode::Benchmark {
+                match_tests,
+                exact_match,
+                seed,
+                property_max_success,
+            }
+        };
+
+        self.compile(options)
+    }
+
     pub fn dump_uplc(&self, blueprint: &Blueprint) -> Result<(), Error> {
         let dir = self.root.join("artifacts");
 
@@ -403,7 +423,7 @@ where
                 property_max_success,
             } => {
                 let tests =
-                    self.collect_tests(verbose, match_tests, exact_match, options.tracing)?;
+                    self.collect_tests(false, match_tests, exact_match, options.tracing)?;
 
                 if !tests.is_empty() {
                     self.event_listener.handle_event(Event::RunningTests);
@@ -442,6 +462,52 @@ where
                     Ok(())
                 }
             }
+            CodeGenMode::Benchmark {
+                match_tests,
+                exact_match,
+                seed,
+                property_max_success,
+            } => {
+                let tests =
+                    self.collect_tests(false, match_tests, exact_match, options.tracing)?;
+
+                if !tests.is_empty() {
+                    self.event_listener.handle_event(Event::RunningTests);
+                }
+
+                let tests = self.run_benchmarks(tests, seed, property_max_success);
+
+                self.checks_count = if tests.is_empty() {
+                    None
+                } else {
+                    Some(tests.iter().fold(0, |acc, test| {
+                        acc + match test {
+                            TestResult::PropertyTestResult(r) => r.iterations,
+                            _ => 1,
+                        }
+                    }))
+                };
+
+                let errors: Vec<Error> = tests
+                    .iter()
+                    .filter_map(|e| {
+                        if e.is_success() {
+                            None
+                        } else {
+                            Some(e.into_error(false))
+                        }
+                    })
+                    .collect();
+
+                self.event_listener
+                    .handle_event(Event::FinishedTests { seed, tests });
+
+                if !errors.is_empty() {
+                    Err(errors)
+                } else {
+                    Ok(())
+                }
+            }
             CodeGenMode::NoOp => Ok(()),
         }
     }
@@ -1020,6 +1086,32 @@ where
             .collect()
     }
 
+    fn run_benchmarks(
+        &self,
+        tests: Vec<Test>,
+        seed: u32,
+        property_max_success: usize,
+    ) -> Vec<TestResult<UntypedExpr, UntypedExpr>> {
+        use rayon::prelude::*;
+
+        let data_types = utils::indexmap::as_ref_values(&self.data_types);
+
+        let plutus_version = &self.config.plutus;
+
+        tests
+            .into_par_iter()
+            .map(|test| match test {
+                Test::UnitTest(unit_test) => unit_test.run(plutus_version),
+                Test::PropertyTest(property_test) => {
+                    property_test.run(seed, property_max_success, plutus_version)
+                }
+            })
+            .collect::<Vec<TestResult<(Constant, Rc<Type>), PlutusData>>>()
+            .into_iter()
+            .map(|test| test.reify(&data_types))
+            .collect()
+    }
+
     fn aiken_files(&mut self, dir: &Path, kind: ModuleKind) -> Result<(), Error> {
         let mut has_default = None;
 

crates/aiken-project/src/options.rs

@@ -28,5 +28,12 @@ pub enum CodeGenMode {
         property_max_success: usize,
     },
     Build(bool),
+    Benchmark {
+        match_tests: Option<Vec<String>>,
+        exact_match: bool,
+        seed: u32,
+        property_max_success: usize
+
+    },
     NoOp,
 }

crates/aiken-project/src/telemetry.rs

@@ -43,10 +43,15 @@ pub enum Event {
         path: PathBuf,
     },
     RunningTests,
+    RunningBenchmarks,
     FinishedTests {
         seed: u32,
         tests: Vec<TestResult<UntypedExpr, UntypedExpr>>,
     },
+    FinishedBenchmarks {
+        seed: u32,
+        tests: Vec<TestResult<UntypedExpr, UntypedExpr>>,
+    },
     WaitingForBuildDirLock,
     ResolvingPackages {
         name: String,

crates/aiken/src/cmd/benchmark.rs

@@ -0,0 +1,61 @@
+use super::build::{filter_traces_parser, trace_level_parser};
+use aiken_lang::ast::{TraceLevel, Tracing};
+use aiken_project::{
+    test_framework::PropertyTest,
+    watch::{self, watch_project, with_project},
+};
+use rand::prelude::*;
+use std::{path::PathBuf, process};
+
+#[derive(clap::Args)]
+/// Type-check an Aiken project
+pub struct Args {
+    /// Path to project
+    directory: Option<PathBuf>,
+
+    /// An initial seed to initialize the pseudo-random generator for property-tests.
+    #[clap(long)]
+    seed: Option<u32>,
+
+    /// Maximum number of successful test run for considering a property-based test valid.
+    #[clap(long, default_value_t = PropertyTest::DEFAULT_MAX_SUCCESS)]
+    max_success: usize,
+
+    /// Only run tests if they match any of these strings.
+    /// You can match a module with `-m aiken/list` or `-m list`.
+    /// You can match a test with `-m "aiken/list.{map}"` or `-m "aiken/option.{flatten_1}"`
+    #[clap(short, long)]
+    match_tests: Option<Vec<String>>,
+
+    /// This is meant to be used with `--match-tests`.
+    /// It forces test names to match exactly
+    #[clap(short, long)]
+    exact_match: bool,
+}
+
+pub fn exec(
+    Args {
+        directory,
+        match_tests,
+        exact_match,
+        seed,
+        max_success,
+    }: Args,
+) -> miette::Result<()> {
+    // Actually we don't want to use check right? 
+    let mut rng = rand::thread_rng();
+
+    let seed = seed.unwrap_or_else(|| rng.gen());
+
+    let result = with_project(directory.as_deref(), false, |p| {
+        // We don't want to check here, we want to benchmark
+        p.benchmark(
+            match_tests.clone(),
+            exact_match,
+            seed,
+            max_success
+        )
+    });
+// todo riley - We need to either print or output the results to a file.
+    result.map_err(|_| process::exit(1))
+}

crates/aiken/src/cmd/mod.rs

@@ -1,6 +1,7 @@
 use aiken_project::config;
 use clap::Parser;
 
+pub mod benchmark;
 pub mod blueprint;
 pub mod build;
 pub mod check;
@@ -35,6 +36,8 @@ pub enum Cmd {
     Docs(docs::Args),
     Add(packages::add::Args),
 
+    Benchmark(benchmark::Args),
+
     #[clap(subcommand)]
     Blueprint(blueprint::Cmd),
 

crates/aiken/src/main.rs

@@ -2,6 +2,7 @@ use aiken_project::{config, pretty};
 #[cfg(not(target_os = "windows"))]
 use cmd::completion;
 use cmd::{
+    benchmark,
     blueprint::{self, address},
     build, check, docs, export, fmt, lsp, new,
     packages::{self, add},
@@ -23,6 +24,7 @@ fn main() -> miette::Result<()> {
         Cmd::Build(args) => build::exec(args),
         Cmd::Address(args) => address::exec(args),
         Cmd::Check(args) => check::exec(args),
+        Cmd::Benchmark(args) => benchmark::exec(args),
         Cmd::Docs(args) => docs::exec(args),
         Cmd::Add(args) => add::exec(args),
         Cmd::Blueprint(args) => blueprint::exec(args),