waalge
/
aiken
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rework benchmarks output 

Going for a terminal plot, for now, as this was the original idea and it is immediately visual. All benchmark points can also be obtained as JSON when redirecting the output, like for tests. So all-in-all, we provide a flexible output which should be useful. Whether it is the best we can do, time (and people/users) will tell.

Signed-off-by: KtorZ <5680256+KtorZ@users.noreply.github.com>
This commit is contained in:
KtorZ 2025-02-09 15:21:45 +01:00
parent 41440f131b
commit b4aa877d6a
No known key found for this signature in database
GPG Key ID: 33173CB6F77F4277
5 changed files with 431 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

439
Cargo.lock generated vendored
View File

File diff suppressed because it is too large Load Diff

75
crates/aiken-lang/src/test_framework.rs
View File

@ -14,7 +14,7 @@ use pallas_primitives::alonzo::{Constr, PlutusData};
use patricia_tree::PatriciaMap;
use std::{
borrow::Borrow,
collections::{BTreeMap, VecDeque},
collections::BTreeMap,
convert::TryFrom,
fmt::{Debug, Display},
ops::Deref,
@ -506,66 +506,6 @@ pub struct Benchmark {
unsafe impl Send for Benchmark {}
trait Sizer {
fn is_done(&self) -> bool;
fn next(&mut self) -> usize;
}
struct FibonacciSizer {
max_size: usize,
previous_sizes: VecDeque<usize>,
current_size: usize,
}
impl FibonacciSizer {
fn new(max_size: usize) -> Self {
Self {
max_size,
previous_sizes: VecDeque::new(),
current_size: 1,
}
}
}
impl Sizer for FibonacciSizer {
fn is_done(&self) -> bool {
self.current_size >= self.max_size
}
fn next(&mut self) -> usize {
match self.previous_sizes.len() {
0 => {
self.previous_sizes.push_front(1);
return 0;
}
1 => {
self.previous_sizes.push_front(1);
return 1;
}
_ => self.current_size += self.previous_sizes.pop_back().unwrap(),
}
self.previous_sizes.push_front(self.current_size);
self.current_size.min(self.max_size)
}
}
#[cfg(test)]
mod test_sizer {
use super::{FibonacciSizer, Sizer};
#[test]
pub fn fib_sizer_sequence() {
let mut sizer = FibonacciSizer::new(100);
let mut sizes = Vec::new();
while !sizer.is_done() {
sizes.push(sizer.next())
}
assert_eq!(sizes, vec![0, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 100])
}
}
impl Benchmark {
pub const DEFAULT_MAX_SIZE: usize = 10;
@ -576,14 +516,15 @@ impl Benchmark {
plutus_version: &PlutusVersion,
) -> BenchmarkResult {
let mut measures = Vec::with_capacity(max_size);
let mut sizer = FibonacciSizer::new(max_size);
let mut prng = Prng::from_seed(seed);
let mut success = true;
let mut size = 0;
while success && !sizer.is_done() {
let size = sizer.next();
let size_as_data = Data::integer(num_bigint::BigInt::from(size));
let fuzzer = self.sampler.program.apply_data(size_as_data);
while success && max_size >= size {
let fuzzer = self
.sampler
.program
.apply_term(&Term::Constant(Constant::Integer(size.into()).into()));
match prng.sample(&fuzzer) {
Ok(None) => {
@ -599,6 +540,8 @@ impl Benchmark {
success = false;
}
}
size += 1;
}
BenchmarkResult {

2
crates/aiken-project/Cargo.toml
View File

@ -42,10 +42,12 @@ pulldown-cmark = { version = "0.12.0", default-features = false, features = [
rayon = "1.7.0"
regex = "1.7.1"
reqwest = { version = "0.11.14", features = ["blocking", "json"] }
rgb = "0.8.50"
semver = { version = "1.0.23", features = ["serde"] }
serde = { version = "1.0.152", features = ["derive"] }
serde_json = { version = "1.0.94", features = ["preserve_order"] }
strip-ansi-escapes = "0.1.1"
textplots = { git = "https://github.com/aiken-lang/textplots-rs.git" }
thiserror = "1.0.39"
tokio = { version = "1.26.0", features = ["full"] }
toml = "0.7.2"

30
crates/aiken-project/src/telemetry.rs
View File

@ -1,6 +1,6 @@
use aiken_lang::{
expr::UntypedExpr,
test_framework::{PropertyTestResult, TestResult, UnitTestResult},
test_framework::{BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult},
};
pub use json::{json_schema, Json};
use std::{
@ -10,6 +10,7 @@ use std::{
path::PathBuf,
};
pub use terminal::Terminal;
use uplc::machine::cost_model::ExBudget;
mod json;
mod terminal;
@ -117,6 +118,18 @@ pub(crate) fn group_by_module(
}
pub(crate) fn find_max_execution_units<T>(xs: &[TestResult<T, T>]) -> (usize, usize, usize) {
fn max_execution_units(max_mem: i64, max_cpu: i64, cost: &ExBudget) -> (i64, i64) {
if cost.mem >= max_mem && cost.cpu >= max_cpu {
(cost.mem, cost.cpu)
} else if cost.mem > max_mem {
(cost.mem, max_cpu)
} else if cost.cpu > max_cpu {
(max_mem, cost.cpu)
} else {
(max_mem, max_cpu)
}
}
let (max_mem, max_cpu, max_iter) =
xs.iter()
.fold((0, 0, 0), |(max_mem, max_cpu, max_iter), test| match test {
@ -124,18 +137,15 @@ pub(crate) fn find_max_execution_units<T>(xs: &[TestResult<T, T>]) -> (usize, us
(max_mem, max_cpu, std::cmp::max(max_iter, *iterations))
}
TestResult::UnitTestResult(UnitTestResult { spent_budget, .. }) => {
if spent_budget.mem >= max_mem && spent_budget.cpu >= max_cpu {
(spent_budget.mem, spent_budget.cpu, max_iter)
} else if spent_budget.mem > max_mem {
(spent_budget.mem, max_cpu, max_iter)
} else if spent_budget.cpu > max_cpu {
(max_mem, spent_budget.cpu, max_iter)
} else {
let (max_mem, max_cpu) = max_execution_units(max_mem, max_cpu, spent_budget);
(max_mem, max_cpu, max_iter)
}
TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => {
let (mut max_mem, mut max_cpu) = (max_mem, max_cpu);
for (_, measure) in measures {
(max_mem, max_cpu) = max_execution_units(max_mem, max_cpu, measure);
}
TestResult::BenchmarkResult(..) => {
unreachable!("unexpected benchmark found amongst test results.")
(max_mem, max_cpu, max_iter)
}
});

131
crates/aiken-project/src/telemetry/terminal.rs
View File

@ -4,11 +4,21 @@ use aiken_lang::{
ast::OnTestFailure,
expr::UntypedExpr,
format::Formatter,
test_framework::{AssertionStyleOptions, PropertyTestResult, TestResult, UnitTestResult},
test_framework::{
AssertionStyleOptions, BenchmarkResult, PropertyTestResult, TestResult, UnitTestResult,
},
};
use owo_colors::{OwoColorize, Stream::Stderr};
use rgb::RGB8;
use std::sync::LazyLock;
use uplc::machine::cost_model::ExBudget;
static BENCH_PLOT_COLOR: LazyLock<RGB8> = LazyLock::new(|| RGB8 {
r: 250,
g: 211,
b: 144,
});
#[derive(Debug, Default, Clone, Copy)]
pub struct Terminal;
@ -224,8 +234,45 @@ impl EventListener for Terminal {
"...".if_supports_color(Stderr, |s| s.bold())
);
}
Event::FinishedBenchmarks { .. } => {
eprintln!("TODO: FinishedBenchmarks");
Event::FinishedBenchmarks { seed, benchmarks } => {
let (max_mem, max_cpu, max_iter) = find_max_execution_units(&benchmarks);
for (module, results) in &group_by_module(&benchmarks) {
let title = module
.if_supports_color(Stderr, |s| s.bold())
.if_supports_color(Stderr, |s| s.blue())
.to_string();
let benchmarks = results
.iter()
.map(|r| fmt_test(r, max_mem, max_cpu, max_iter, true))
.collect::<Vec<String>>()
.join("\n");
let seed_info = format!(
"with {opt}={seed}",
opt = "--seed".if_supports_color(Stderr, |s| s.bold()),
seed = format!("{seed}").if_supports_color(Stderr, |s| s.bold())
);
if !benchmarks.is_empty() {
println!();
}
println!(
"{}\n",
pretty::indent(
&pretty::open_box(&title, &benchmarks, &seed_info, |border| border
.if_supports_color(Stderr, |s| s.bright_black())
.to_string()),
4
)
);
}
if !benchmarks.is_empty() {
println!();
}
}
}
}
@ -239,7 +286,9 @@ fn fmt_test(
styled: bool,
) -> String {
// Status
let mut test = if result.is_success() {
let mut test = if matches!(result, TestResult::BenchmarkResult { .. }) {
String::new()
} else if result.is_success() {
pretty::style_if(styled, "PASS".to_string(), |s| {
s.if_supports_color(Stderr, |s| s.bold())
.if_supports_color(Stderr, |s| s.green())
@ -285,18 +334,73 @@ fn fmt_test(
if *iterations > 1 { "s" } else { "" }
);
}
TestResult::BenchmarkResult(..) => {
unreachable!("unexpected benchmark found amongst test results.")
TestResult::BenchmarkResult(BenchmarkResult { measures, .. }) => {
let max_size = measures
.iter()
.map(|(size, _)| *size)
.max()
.unwrap_or_default();
let mem_chart = format!(
"{title}\n{chart}",
title = "memory units"
.if_supports_color(Stderr, |s| s.yellow())
.if_supports_color(Stderr, |s| s.bold()),
chart = plot(
&BENCH_PLOT_COLOR,
measures
.iter()
.map(|(size, budget)| (*size as f32, budget.mem as f32))
.collect::<Vec<_>>(),
max_size
)
);
let cpu_chart = format!(
"{title}\n{chart}",
title = "cpu units"
.if_supports_color(Stderr, |s| s.yellow())
.if_supports_color(Stderr, |s| s.bold()),
chart = plot(
&BENCH_PLOT_COLOR,
measures
.iter()
.map(|(size, budget)| (*size as f32, budget.cpu as f32))
.collect::<Vec<_>>(),
max_size
)
);
let charts = mem_chart
.lines()
.zip(cpu_chart.lines())
.map(|(l, r)| format!(" {}{r}", pretty::pad_right(l.to_string(), 55, " ")))
.collect::<Vec<_>>()
.join("\n");
test = format!("{test}{charts}",);
}
}
// Title
test = format!(
test = match result {
TestResult::BenchmarkResult(..) => {
format!(
"{title}\n{test}\n",
title = pretty::style_if(styled, result.title().to_string(), |s| s
.if_supports_color(Stderr, |s| s.bright_blue())
.to_string())
)
}
TestResult::UnitTestResult(..) | TestResult::PropertyTestResult(..) => {
format!(
"{test} {title}",
title = pretty::style_if(styled, result.title().to_string(), |s| s
.if_supports_color(Stderr, |s| s.bright_blue())
.to_string())
);
)
}
};
// Annotations
match result {
@ -452,3 +556,14 @@ fn fmt_test_summary<T>(tests: &[&TestResult<T, T>], styled: bool) -> String {
.to_string()),
)
}
fn plot(color: &RGB8, points: Vec<(f32, f32)>, max_size: usize) -> String {
use textplots::{Chart, ColorPlot, Shape};
let mut chart = Chart::new(80, 50, 1.0, max_size as f32);
let plot = Shape::Lines(&points);
let chart = chart.linecolorplot(&plot, *color);
chart.borders();
chart.axis();
chart.figures();
chart.to_string()
}