rework benchmarks output

Going for a terminal plot, for now, as this was the original idea and it is immediately visual. All benchmark points can also be obtained as JSON when redirecting the output, like for tests. So all-in-all, we provide a flexible output which should be useful. Whether it is the best we can do, time (and people/users) will tell.

Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com>

crates/aiken-lang/src/test_framework.rs

@@ -14,7 +14,7 @@ use pallas_primitives::alonzo::{Constr, PlutusData};
 use patricia_tree::PatriciaMap;
 use std::{
     borrow::Borrow,
-    collections::{BTreeMap, VecDeque},
+    collections::BTreeMap,
     convert::TryFrom,
     fmt::{Debug, Display},
     ops::Deref,
@@ -506,66 +506,6 @@ pub struct Benchmark {
 
 unsafe impl Send for Benchmark {}
 
-trait Sizer {
-    fn is_done(&self) -> bool;
-    fn next(&mut self) -> usize;
-}
-
-struct FibonacciSizer {
-    max_size: usize,
-    previous_sizes: VecDeque<usize>,
-    current_size: usize,
-}
-
-impl FibonacciSizer {
-    fn new(max_size: usize) -> Self {
-        Self {
-            max_size,
-            previous_sizes: VecDeque::new(),
-            current_size: 1,
-        }
-    }
-}
-
-impl Sizer for FibonacciSizer {
-    fn is_done(&self) -> bool {
-        self.current_size >= self.max_size
-    }
-
-    fn next(&mut self) -> usize {
-        match self.previous_sizes.len() {
-            0 => {
-                self.previous_sizes.push_front(1);
-                return 0;
-            }
-            1 => {
-                self.previous_sizes.push_front(1);
-                return 1;
-            }
-            _ => self.current_size += self.previous_sizes.pop_back().unwrap(),
-        }
-
-        self.previous_sizes.push_front(self.current_size);
-
-        self.current_size.min(self.max_size)
-    }
-}
-
-#[cfg(test)]
-mod test_sizer {
-    use super::{FibonacciSizer, Sizer};
-
-    #[test]
-    pub fn fib_sizer_sequence() {
-        let mut sizer = FibonacciSizer::new(100);
-        let mut sizes = Vec::new();
-        while !sizer.is_done() {
-            sizes.push(sizer.next())
-        }
-        assert_eq!(sizes, vec![0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 100])
-    }
-}
-
 impl Benchmark {
     pub const DEFAULT_MAX_SIZE: usize = 10;
 
@@ -576,14 +516,15 @@ impl Benchmark {
         plutus_version: &PlutusVersion,
     ) -> BenchmarkResult {
         let mut measures = Vec::with_capacity(max_size);
-        let mut sizer = FibonacciSizer::new(max_size);
         let mut prng = Prng::from_seed(seed);
         let mut success = true;
+        let mut size = 0;
 
-        while success && !sizer.is_done() {
+        while success && max_size >= size {
-            let size = sizer.next();
+            let fuzzer = self
-            let size_as_data = Data::integer(num_bigint::BigInt::from(size));
+                .sampler
-            let fuzzer = self.sampler.program.apply_data(size_as_data);
+                .program
+                .apply_term(&Term::Constant(Constant::Integer(size.into()).into()));
 
             match prng.sample(&fuzzer) {
                 Ok(None) => {
@@ -599,6 +540,8 @@ impl Benchmark {
                     success = false;
                 }
             }
+
+            size += 1;
         }
 
         BenchmarkResult {

crates/aiken-project/Cargo.toml

@@ -42,10 +42,12 @@ pulldown-cmark = { version = "0.12.0", default-features = false, features = [
 rayon = "1.7.0"
 regex = "1.7.1"
 reqwest = { version = "0.11.14", features = ["blocking", "json"] }
+rgb = "0.8.50"
 semver = { version = "1.0.23", features = ["serde"] }
 serde = { version = "1.0.152", features = ["derive"] }
 serde_json = { version = "1.0.94", features = ["preserve_order"] }
 strip-ansi-escapes = "0.1.1"
+textplots = { git = "https://github.com/aiken-lang/textplots-rs.git" }
 thiserror = "1.0.39"
 tokio = { version = "1.26.0", features = ["full"] }
 toml = "0.7.2"

crates/aiken-project/src/telemetry.rs

@@ -1,6 +1,6 @@
 use aiken_lang::{
     expr::UntypedExpr,
-    test_framework::{PropertyTestResult, TestResult, UnitTestResult},
+    test_framework::{BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult},
 };
 pub use json::{json_schema, Json};
 use std::{
@@ -10,6 +10,7 @@ use std::{
     path::PathBuf,
 };
 pub use terminal::Terminal;
+use uplc::machine::cost_model::ExBudget;
 
 mod json;
 mod terminal;
@@ -117,6 +118,18 @@ pub(crate) fn group_by_module(
 }
 
 pub(crate) fn find_max_execution_units<T>(xs: &[TestResult<T, T>]) -> (usize, usize, usize) {
+    fn max_execution_units(max_mem: i64, max_cpu: i64, cost: &ExBudget) -> (i64, i64) {
+        if cost.mem >= max_mem && cost.cpu >= max_cpu {
+            (cost.mem, cost.cpu)
+        } else if cost.mem > max_mem {
+            (cost.mem, max_cpu)
+        } else if cost.cpu > max_cpu {
+            (max_mem, cost.cpu)
+        } else {
+            (max_mem, max_cpu)
+        }
+    }
+
     let (max_mem, max_cpu, max_iter) =
         xs.iter()
             .fold((0, 0, 0), |(max_mem, max_cpu, max_iter), test| match test {
@@ -124,18 +137,15 @@ pub(crate) fn find_max_execution_units<T>(xs: &[TestResult<T, T>]) -> (usize, us
                     (max_mem, max_cpu, std::cmp::max(max_iter, *iterations))
                 }
                 TestResult::UnitTestResult(UnitTestResult { spent_budget, .. }) => {
-                    if spent_budget.mem >= max_mem && spent_budget.cpu >= max_cpu {
+                    let (max_mem, max_cpu) = max_execution_units(max_mem, max_cpu, spent_budget);
-                        (spent_budget.mem, spent_budget.cpu, max_iter)
+                    (max_mem, max_cpu, max_iter)
-                    } else if spent_budget.mem > max_mem {
-                        (spent_budget.mem, max_cpu, max_iter)
-                    } else if spent_budget.cpu > max_cpu {
-                        (max_mem, spent_budget.cpu, max_iter)
-                    } else {
-                        (max_mem, max_cpu, max_iter)
-                    }
                 }
-                TestResult::BenchmarkResult(..) => {
+                TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => {
-                    unreachable!("unexpected benchmark found amongst test results.")
+                    let (mut max_mem, mut max_cpu) = (max_mem, max_cpu);
+                    for (_, measure) in measures {
+                        (max_mem, max_cpu) = max_execution_units(max_mem, max_cpu, measure);
+                    }
+                    (max_mem, max_cpu, max_iter)
                 }
             });
 

crates/aiken-project/src/telemetry/terminal.rs

@@ -4,11 +4,21 @@ use aiken_lang::{
     ast::OnTestFailure,
     expr::UntypedExpr,
     format::Formatter,
-    test_framework::{AssertionStyleOptions, PropertyTestResult, TestResult, UnitTestResult},
+    test_framework::{
+        AssertionStyleOptions, BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult,
+    },
 };
 use owo_colors::{OwoColorize, Stream::Stderr};
+use rgb::RGB8;
+use std::sync::LazyLock;
 use uplc::machine::cost_model::ExBudget;
 
+static BENCH_PLOT_COLOR: LazyLock<RGB8> = LazyLock::new(|| RGB8 {
+    r: 250,
+    g: 211,
+    b: 144,
+});
+
 #[derive(Debug, Default, Clone, Copy)]
 pub struct Terminal;
 
@@ -224,8 +234,45 @@ impl EventListener for Terminal {
                     "...".if_supports_color(Stderr, |s| s.bold())
                 );
             }
-            Event::FinishedBenchmarks { .. } => {
+            Event::FinishedBenchmarks { seed, benchmarks } => {
-                eprintln!("TODO: FinishedBenchmarks");
+                let (max_mem, max_cpu, max_iter) = find_max_execution_units(&benchmarks);
+
+                for (module, results) in &group_by_module(&benchmarks) {
+                    let title = module
+                        .if_supports_color(Stderr, |s| s.bold())
+                        .if_supports_color(Stderr, |s| s.blue())
+                        .to_string();
+
+                    let benchmarks = results
+                        .iter()
+                        .map(|r| fmt_test(r, max_mem, max_cpu, max_iter, true))
+                        .collect::<Vec<String>>()
+                        .join("\n");
+
+                    let seed_info = format!(
+                        "with {opt}={seed}",
+                        opt = "--seed".if_supports_color(Stderr, |s| s.bold()),
+                        seed = format!("{seed}").if_supports_color(Stderr, |s| s.bold())
+                    );
+
+                    if !benchmarks.is_empty() {
+                        println!();
+                    }
+
+                    println!(
+                        "{}\n",
+                        pretty::indent(
+                            &pretty::open_box(&title, &benchmarks, &seed_info, |border| border
+                                .if_supports_color(Stderr, |s| s.bright_black())
+                                .to_string()),
+                            4
+                        )
+                    );
+                }
+
+                if !benchmarks.is_empty() {
+                    println!();
+                }
             }
         }
     }
@@ -239,7 +286,9 @@ fn fmt_test(
     styled: bool,
 ) -> String {
     // Status
-    let mut test = if result.is_success() {
+    let mut test = if matches!(result, TestResult::BenchmarkResult { .. }) {
+        String::new()
+    } else if result.is_success() {
         pretty::style_if(styled, "PASS".to_string(), |s| {
             s.if_supports_color(Stderr, |s| s.bold())
                 .if_supports_color(Stderr, |s| s.green())
@@ -285,18 +334,73 @@ fn fmt_test(
                 if *iterations > 1 { "s" } else { "" }
             );
         }
-        TestResult::BenchmarkResult(..) => {
+        TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => {
-            unreachable!("unexpected benchmark found amongst test results.")
+            let max_size = measures
+                .iter()
+                .map(|(size, _)| *size)
+                .max()
+                .unwrap_or_default();
+
+            let mem_chart = format!(
+                "{title}\n{chart}",
+                title = "memory units"
+                    .if_supports_color(Stderr, |s| s.yellow())
+                    .if_supports_color(Stderr, |s| s.bold()),
+                chart = plot(
+                    &BENCH_PLOT_COLOR,
+                    measures
+                        .iter()
+                        .map(|(size, budget)| (*size as f32, budget.mem as f32))
+                        .collect::<Vec<_>>(),
+                    max_size
+                )
+            );
+
+            let cpu_chart = format!(
+                "{title}\n{chart}",
+                title = "cpu units"
+                    .if_supports_color(Stderr, |s| s.yellow())
+                    .if_supports_color(Stderr, |s| s.bold()),
+                chart = plot(
+                    &BENCH_PLOT_COLOR,
+                    measures
+                        .iter()
+                        .map(|(size, budget)| (*size as f32, budget.cpu as f32))
+                        .collect::<Vec<_>>(),
+                    max_size
+                )
+            );
+
+            let charts = mem_chart
+                .lines()
+                .zip(cpu_chart.lines())
+                .map(|(l, r)| format!("  {}{r}", pretty::pad_right(l.to_string(), 55, " ")))
+                .collect::<Vec<_>>()
+                .join("\n");
+
+            test = format!("{test}{charts}",);
         }
     }
 
     // Title
-    test = format!(
+    test = match result {
-        "{test} {title}",
+        TestResult::BenchmarkResult(..) => {
-        title = pretty::style_if(styled, result.title().to_string(), |s| s
+            format!(
-            .if_supports_color(Stderr, |s| s.bright_blue())
+                "{title}\n{test}\n",
-            .to_string())
+                title = pretty::style_if(styled, result.title().to_string(), |s| s
-    );
+                    .if_supports_color(Stderr, |s| s.bright_blue())
+                    .to_string())
+            )
+        }
+        TestResult::UnitTestResult(..) | TestResult::PropertyTestResult(..) => {
+            format!(
+                "{test} {title}",
+                title = pretty::style_if(styled, result.title().to_string(), |s| s
+                    .if_supports_color(Stderr, |s| s.bright_blue())
+                    .to_string())
+            )
+        }
+    };
 
     // Annotations
     match result {
@@ -452,3 +556,14 @@ fn fmt_test_summary<T>(tests: &[&TestResult<T, T>], styled: bool) -> String {
             .to_string()),
     )
 }
+
+fn plot(color: &RGB8, points: Vec<(f32, f32)>, max_size: usize) -> String {
+    use textplots::{Chart, ColorPlot, Shape};
+    let mut chart = Chart::new(80, 50, 1.0, max_size as f32);
+    let plot = Shape::Lines(&points);
+    let chart = chart.linecolorplot(&plot, *color);
+    chart.borders();
+    chart.axis();
+    chart.figures();
+    chart.to_string()
+}