waalge
/
aiken
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Addressed comments on benchmarking PR

This commit is contained in:
Riley-Kilgore 2025-01-14 05:16:08 -08:00 committed by Riley
parent df05ae7e5d
commit bd44b22d59
17 changed files with 253 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
crates/aiken-lang/src/format.rs
View File

@ -604,8 +604,9 @@ impl<'comments> Formatter<'comments> {
}
#[allow(clippy::too_many_arguments)]
fn definition_test<'a>(
fn definition_test_or_bench<'a>(
&mut self,
keyword: &'static str,
name: &'a str,
args: &'a [UntypedArgVia],
body: &'a UntypedExpr,
@ -613,14 +614,19 @@ impl<'comments> Formatter<'comments> {
on_test_failure: &'a OnTestFailure,
) -> Document<'a> {
// Fn name and args
let head = "test "
let head = keyword
.to_doc()
.append(" ")
.append(name)
.append(wrap_args(args.iter().map(|e| (self.fn_arg_via(e), false))))
.append(match on_test_failure {
OnTestFailure::FailImmediately => "",
OnTestFailure::SucceedEventually => " fail",
OnTestFailure::SucceedImmediately => " fail once",
.append(if keyword == "test" {
match on_test_failure {
OnTestFailure::FailImmediately => "",
OnTestFailure::SucceedEventually => " fail",
OnTestFailure::SucceedImmediately => " fail once",
}
} else {
""
})
.group();
@ -640,6 +646,18 @@ impl<'comments> Formatter<'comments> {
.append("}")
}
#[allow(clippy::too_many_arguments)]
fn definition_test<'a>(
&mut self,
name: &'a str,
args: &'a [UntypedArgVia],
body: &'a UntypedExpr,
end_location: usize,
on_test_failure: &'a OnTestFailure,
) -> Document<'a> {
self.definition_test_or_bench("test", name, args, body, end_location, on_test_failure)
}
#[allow(clippy::too_many_arguments)]
fn definition_benchmark<'a>(
&mut self,
@ -649,32 +667,7 @@ impl<'comments> Formatter<'comments> {
end_location: usize,
on_test_failure: &'a OnTestFailure,
) -> Document<'a> {
// Fn name and args
let head = "bench "
.to_doc()
.append(name)
.append(wrap_args(args.iter().map(|e| (self.fn_arg_via(e), false))))
.append(match on_test_failure {
OnTestFailure::FailImmediately => "",
OnTestFailure::SucceedEventually => "",
OnTestFailure::SucceedImmediately => "",
})
.group();
// Format body
let body = self.expr(body, true);
// Add any trailing comments
let body = match printed_comments(self.pop_comments(end_location), false) {
Some(comments) => body.append(line()).append(comments),
None => body,
};
// Stick it all together
head.append(" {")
.append(line().append(body).nest(INDENT).group())
.append(line())
.append("}")
self.definition_test_or_bench("bench", name, args, body, end_location, on_test_failure)
}
fn definition_validator<'a>(

123
crates/aiken-lang/src/parser/definition/benchmark.rs
View File

@ -1,130 +1,11 @@
use crate::{
ast,
ast::OnTestFailure,
expr::UntypedExpr,
parser::{
annotation,
chain::{call::parser as call, field_access, tuple_index::parser as tuple_index, Chain},
error::ParseError,
expr::{self, bytearray, int as uint, list, string, tuple, var},
pattern,
token::Token,
},
parser::{error::ParseError, token::Token},
};
use chumsky::prelude::*;
pub fn parser() -> impl Parser<Token, ast::UntypedDefinition, Error = ParseError> {
just(Token::Benchmark)
.ignore_then(select! {Token::Name {name} => name})
.then(
via()
.separated_by(just(Token::Comma))
.allow_trailing()
.delimited_by(just(Token::LeftParen), just(Token::RightParen)),
)
.then(
just(Token::Fail)
.ignore_then(just(Token::Once).ignored().or_not().map(|once| {
once.map(|_| OnTestFailure::SucceedImmediately)
.unwrap_or(OnTestFailure::SucceedEventually)
}))
.or_not(),
)
.map_with_span(|name, span| (name, span))
.then(
expr::sequence()
.or_not()
.delimited_by(just(Token::LeftBrace), just(Token::RightBrace)),
)
.map_with_span(|((((name, arguments), fail), span_end), body), span| {
ast::UntypedDefinition::Benchmark(ast::Function {
arguments,
body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)),
doc: None,
location: span_end,
end_position: span.end - 1,
name,
public: false,
return_annotation: None,
return_type: (),
on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately),
})
})
}
pub fn via() -> impl Parser<Token, ast::UntypedArgVia, Error = ParseError> {
choice((
select! {Token::DiscardName {name} => name}.map_with_span(|name, span| {
ast::ArgBy::ByName(ast::ArgName::Discarded {
label: name.clone(),
name,
location: span,
})
}),
select! {Token::Name {name} => name}.map_with_span(|name, location| {
ast::ArgBy::ByName(ast::ArgName::Named {
label: name.clone(),
name,
location,
})
}),
pattern().map(ast::ArgBy::ByPattern),
))
.then(just(Token::Colon).ignore_then(annotation()).or_not())
.map_with_span(|(arg_name, annotation), location| (arg_name, annotation, location))
.then_ignore(just(Token::Via))
.then(fuzzer())
.map(|((by, annotation, location), via)| ast::ArgVia {
arg: ast::UntypedArg {
by,
annotation,
location,
doc: None,
is_validator_param: false,
},
via,
})
}
pub fn fuzzer<'a>() -> impl Parser<Token, UntypedExpr, Error = ParseError> + 'a {
recursive(|expression| {
let chain = choice((
tuple_index(),
field_access::parser(),
call(expression.clone()),
));
let int = || {
just(Token::Minus)
.to(ast::UnOp::Negate)
.map_with_span(|op, span| (op, span))
.or_not()
.then(uint())
.map(|(op, value)| match op {
None => value,
Some((op, location)) => UntypedExpr::UnOp {
op,
location,
value: Box::new(value),
},
})
};
choice((
int(),
string(),
bytearray(),
tuple(expression.clone()),
list(expression.clone()),
var(),
))
.then(chain.repeated())
.foldl(|expr, chain| match chain {
Chain::Call(args, span) => expr.call(args, span),
Chain::FieldAccess(label, span) => expr.field_access(label, span),
Chain::TupleIndex(index, span) => expr.tuple_index(index, span),
})
})
crate::parser::definition::test_like::parser(Token::Benchmark)
}
#[cfg(test)]

1
crates/aiken-lang/src/parser/definition/mod.rs
View File

@ -6,6 +6,7 @@ mod data_type;
mod function;
pub mod import;
mod test;
pub mod test_like;
mod type_alias;
mod validator;

121
crates/aiken-lang/src/parser/definition/test.rs
View File

@ -1,131 +1,14 @@
use crate::{
ast,
ast::OnTestFailure,
expr::UntypedExpr,
parser::{
annotation,
chain::{call::parser as call, field_access, tuple_index::parser as tuple_index, Chain},
error::ParseError,
expr::{self, bytearray, int as uint, list, string, tuple, var},
pattern,
token::Token,
},
parser::{error::ParseError, token::Token},
};
use chumsky::prelude::*;
pub fn parser() -> impl Parser<Token, ast::UntypedDefinition, Error = ParseError> {
just(Token::Test)
.ignore_then(select! {Token::Name {name} => name})
.then(
via()
.separated_by(just(Token::Comma))
.allow_trailing()
.delimited_by(just(Token::LeftParen), just(Token::RightParen)),
)
.then(
just(Token::Fail)
.ignore_then(just(Token::Once).ignored().or_not().map(|once| {
once.map(|_| OnTestFailure::SucceedImmediately)
.unwrap_or(OnTestFailure::SucceedEventually)
}))
.or_not(),
)
.map_with_span(|name, span| (name, span))
.then(
expr::sequence()
.or_not()
.delimited_by(just(Token::LeftBrace), just(Token::RightBrace)),
)
.map_with_span(|((((name, arguments), fail), span_end), body), span| {
ast::UntypedDefinition::Test(ast::Function {
arguments,
body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)),
doc: None,
location: span_end,
end_position: span.end - 1,
name,
public: false,
return_annotation: None,
return_type: (),
on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately),
})
})
crate::parser::definition::test_like::parser(Token::Test)
}
pub fn via() -> impl Parser<Token, ast::UntypedArgVia, Error = ParseError> {
choice((
select! {Token::DiscardName {name} => name}.map_with_span(|name, span| {
ast::ArgBy::ByName(ast::ArgName::Discarded {
label: name.clone(),
name,
location: span,
})
}),
select! {Token::Name {name} => name}.map_with_span(|name, location| {
ast::ArgBy::ByName(ast::ArgName::Named {
label: name.clone(),
name,
location,
})
}),
pattern().map(ast::ArgBy::ByPattern),
))
.then(just(Token::Colon).ignore_then(annotation()).or_not())
.map_with_span(|(arg_name, annotation), location| (arg_name, annotation, location))
.then_ignore(just(Token::Via))
.then(fuzzer())
.map(|((by, annotation, location), via)| ast::ArgVia {
arg: ast::UntypedArg {
by,
annotation,
location,
doc: None,
is_validator_param: false,
},
via,
})
}
pub fn fuzzer<'a>() -> impl Parser<Token, UntypedExpr, Error = ParseError> + 'a {
recursive(|expression| {
let chain = choice((
tuple_index(),
field_access::parser(),
call(expression.clone()),
));
let int = || {
just(Token::Minus)
.to(ast::UnOp::Negate)
.map_with_span(|op, span| (op, span))
.or_not()
.then(uint())
.map(|(op, value)| match op {
None => value,
Some((op, location)) => UntypedExpr::UnOp {
op,
location,
value: Box::new(value),
},
})
};
choice((
int(),
string(),
bytearray(),
tuple(expression.clone()),
list(expression.clone()),
var(),
))
.then(chain.repeated())
.foldl(|expr, chain| match chain {
Chain::Call(args, span) => expr.call(args, span),
Chain::FieldAccess(label, span) => expr.field_access(label, span),
Chain::TupleIndex(index, span) => expr.tuple_index(index, span),
})
})
}
#[cfg(test)]
mod tests {

143
crates/aiken-lang/src/parser/definition/test_like.rs Normal file
View File

@ -0,0 +1,143 @@
use crate::{
ast,
ast::OnTestFailure,
expr::UntypedExpr,
parser::{
annotation,
chain::{call::parser as call, field_access, tuple_index::parser as tuple_index, Chain},
error::ParseError,
expr::{self, bytearray, int as uint, list, string, tuple, var},
pattern,
token::Token,
},
};
use chumsky::prelude::*;
pub fn parser(keyword: Token) -> impl Parser<Token, ast::UntypedDefinition, Error = ParseError> {
just(keyword.clone())
.ignore_then(select! {Token::Name {name} => name})
.then(
via()
.separated_by(just(Token::Comma))
.allow_trailing()
.delimited_by(just(Token::LeftParen), just(Token::RightParen)),
)
.then(
just(Token::Fail)
.ignore_then(just(Token::Once).ignored().or_not().map(|once| {
once.map(|_| OnTestFailure::SucceedImmediately)
.unwrap_or(OnTestFailure::SucceedEventually)
}))
.or_not(),
)
.map_with_span(|name, span| (name, span))
.then(
expr::sequence()
.or_not()
.delimited_by(just(Token::LeftBrace), just(Token::RightBrace)),
)
.map_with_span(move |((((name, arguments), fail), span_end), body), span| {
match keyword {
Token::Test => ast::UntypedDefinition::Test(ast::Function {
arguments,
body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)),
doc: None,
location: span_end,
end_position: span.end - 1,
name,
public: false,
return_annotation: None,
return_type: (),
on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately),
}),
Token::Benchmark => ast::UntypedDefinition::Benchmark(ast::Function {
arguments,
body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)),
doc: None,
location: span_end,
end_position: span.end - 1,
name,
public: false,
return_annotation: None,
return_type: (),
on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately),
}),
_ => unreachable!("Only Test and Benchmark tokens are supported"),
}
})
}
pub fn via() -> impl Parser<Token, ast::UntypedArgVia, Error = ParseError> {
choice((
select! {Token::DiscardName {name} => name}.map_with_span(|name, span| {
ast::ArgBy::ByName(ast::ArgName::Discarded {
label: name.clone(),
name,
location: span,
})
}),
select! {Token::Name {name} => name}.map_with_span(|name, location| {
ast::ArgBy::ByName(ast::ArgName::Named {
label: name.clone(),
name,
location,
})
}),
pattern().map(ast::ArgBy::ByPattern),
))
.then(just(Token::Colon).ignore_then(annotation()).or_not())
.map_with_span(|(arg_name, annotation), location| (arg_name, annotation, location))
.then_ignore(just(Token::Via))
.then(fuzzer())
.map(|((by, annotation, location), via)| ast::ArgVia {
arg: ast::UntypedArg {
by,
annotation,
location,
doc: None,
is_validator_param: false,
},
via,
})
}
pub fn fuzzer<'a>() -> impl Parser<Token, UntypedExpr, Error = ParseError> + 'a {
recursive(|expression| {
let chain = choice((
tuple_index(),
field_access::parser(),
call(expression.clone()),
));
let int = || {
just(Token::Minus)
.to(ast::UnOp::Negate)
.map_with_span(|op, span| (op, span))
.or_not()
.then(uint())
.map(|(op, value)| match op {
None => value,
Some((op, location)) => UntypedExpr::UnOp {
op,
location,
value: Box::new(value),
},
})
};
choice((
int(),
string(),
bytearray(),
tuple(expression.clone()),
list(expression.clone()),
var(),
))
.then(chain.repeated())
.foldl(|expr, chain| match chain {
Chain::Call(args, span) => expr.call(args, span),
Chain::FieldAccess(label, span) => expr.field_access(label, span),
Chain::TupleIndex(index, span) => expr.tuple_index(index, span),
})
})
}

22
crates/aiken-lang/src/test_framework.rs
View File

@ -382,7 +382,7 @@ impl PropertyTest {
let mut counterexample = Counterexample {
value,
choices: next_prng.choices(),
cache: Cache::new(move |choices| {
cache: Cache::new(|choices| {
match Prng::from_choices(choices).sample(&self.fuzzer.program) {
Err(..) => Status::Invalid,
Ok(None) => Status::Invalid,
@ -442,6 +442,18 @@ impl PropertyTest {
/// ----- Benchmark -----------------------------------------------------------------
#[derive(Debug, Clone)]
pub struct Sampler<T> {
pub program: Program<T>,
pub type_info: Rc<Type>,
/// A version of the Fuzzer's type that has gotten rid of
/// all erasable opaque type. This is needed in order to
/// generate Plutus data with the appropriate shape.
pub stripped_type_info: Rc<Type>,
}
#[derive(Debug, Clone)]
pub struct Benchmark {
pub input_path: PathBuf,
@ -449,7 +461,7 @@ pub struct Benchmark {
pub name: String,
pub on_test_failure: OnTestFailure,
pub program: Program<Name>,
pub sampler: Fuzzer<Name>,
pub sampler: Sampler<Name>,
}
unsafe impl Send for Benchmark {}
@ -458,14 +470,14 @@ impl Benchmark {
pub fn benchmark(
self,
seed: u32,
n: usize,
max_iterations: usize,
plutus_version: &PlutusVersion,
) -> Vec<BenchmarkResult> {
let mut results = Vec::with_capacity(n);
let mut results = Vec::with_capacity(max_iterations);
let mut iteration = 0;
let mut prng = Prng::from_seed(seed);
while n > iteration {
while max_iterations > iteration {
let fuzzer = self
.sampler
.program

17
crates/aiken-lang/src/tipo/error.rs
View File

@ -1452,6 +1452,21 @@ fn suggest_unify(
expected,
given
},
Some(UnifyErrorSituation::SamplerAnnotationMismatch) => formatdoc! {
r#"While comparing the return annotation of a Sampler with its actual return type, I realized that both don't match.
I am inferring the Sampler should return:
{}
but I found a conflicting annotation saying it returns:
{}
Either, fix (or remove) the annotation or adjust the Sampler to return the expected type."#,
expected,
given
},
None => formatdoc! {
r#"I am inferring the following type:
@ -1883,6 +1898,8 @@ pub enum UnifyErrorSituation {
Operator(BinOp),
FuzzerAnnotationMismatch,
SamplerAnnotationMismatch,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]

16
crates/aiken-lang/src/tipo/infer.rs
View File

@ -358,15 +358,12 @@ fn infer_definition(
.map(|ann| hydrator.type_from_annotation(ann, environment))
.transpose()?;
let (inferred_annotation, inferred_inner_type) = match infer_fuzzer(
let (inferred_annotation, inferred_inner_type) = infer_fuzzer(
environment,
provided_inner_type.clone(),
&typed_via.tipo(),
&arg.via.location(),
) {
Ok(result) => Ok(result),
Err(err) => Err(err),
}?;
)?;
// Ensure that the annotation, if any, matches the type inferred from the
// Fuzzer.
@ -494,15 +491,12 @@ fn infer_definition(
.map(|ann| hydrator.type_from_annotation(ann, environment))
.transpose()?;
let (inferred_annotation, inferred_inner_type) = match infer_sampler(
let (inferred_annotation, inferred_inner_type) = infer_sampler(
environment,
provided_inner_type.clone(),
&typed_via.tipo(),
&arg.via.location(),
) {
Ok(result) => Ok(result),
Err(err) => Err(err),
}?;
)?;
// Ensure that the annotation, if any, matches the type inferred from the
// Fuzzer.
@ -518,7 +512,7 @@ fn infer_definition(
location: arg.arg.location,
expected: inferred_inner_type.clone(),
given: provided_inner_type.clone(),
situation: Some(UnifyErrorSituation::FuzzerAnnotationMismatch),
situation: Some(UnifyErrorSituation::SamplerAnnotationMismatch),
rigid_type_names: hydrator.rigid_names(),
});
}

47
crates/aiken-project/src/lib.rs
View File

@ -304,7 +304,6 @@ where
seed: u32,
times_to_run: usize,
env: Option<String>,
output: PathBuf,
) -> Result<(), Vec<Error>> {
let options = Options {
tracing: Tracing::silent(),
@ -314,7 +313,6 @@ where
exact_match,
seed,
times_to_run,
output,
},
blueprint_path: self.blueprint_path(None),
};
@ -427,7 +425,7 @@ where
seed,
property_max_success,
} => {
let tests = self.collect_tests(false, match_tests, exact_match, options.tracing)?;
let tests = self.collect_tests(verbose, match_tests, exact_match, options.tracing)?;
if !tests.is_empty() {
self.event_listener.handle_event(Event::RunningTests);
@ -471,9 +469,7 @@ where
exact_match,
seed,
times_to_run,
output,
} => {
// todo - collect benchmarks
let tests =
self.collect_benchmarks(false, match_tests, exact_match, options.tracing)?;
@ -496,51 +492,12 @@ where
self.event_listener.handle_event(Event::FinishedBenchmarks {
seed,
tests: tests.clone(),
tests,
});
if !errors.is_empty() {
Err(errors)
} else {
// Write benchmark results to CSV
use std::fs::File;
use std::io::Write;
let mut writer = File::create(&output).map_err(|error| {
vec![Error::FileIo {
error,
path: output.clone(),
}]
})?;
// Write CSV header
writeln!(writer, "test_name,module,memory,cpu").map_err(|error| {
vec![Error::FileIo {
error,
path: output.clone(),
}]
})?;
// Write benchmark results
for test in tests {
if let TestResult::Benchmark(result) = test {
writeln!(
writer,
"{},{},{},{}",
result.test.name,
result.test.module,
result.cost.mem,
result.cost.cpu
)
.map_err(|error| {
vec![Error::FileIo {
error,
path: output.clone(),
}]
})?;
}
}
Ok(())
}
}

1
crates/aiken-project/src/options.rs
View File

@ -34,7 +34,6 @@ pub enum CodeGenMode {
exact_match: bool,
seed: u32,
times_to_run: usize,
output: PathBuf,
},
NoOp,
}

1
crates/aiken-project/src/telemetry.rs
View File

@ -135,7 +135,6 @@ pub(crate) fn find_max_execution_units<T>(xs: &[TestResult<T, T>]) -> (usize, us
}
}
TestResult::Benchmark(..) => {
// todo riley - should this be reachable?
unreachable!("property returned benchmark result ?!")
}
});

24
crates/aiken-project/src/telemetry/json.rs
View File

@ -39,6 +39,30 @@ impl EventListener for Json {
});
println!("{}", serde_json::to_string_pretty(&json_output).unwrap());
}
Event::FinishedBenchmarks { tests, seed } => {
let benchmark_results: Vec<_> = tests
.into_iter()
.filter_map(|test| {
if let TestResult::Benchmark(result) = test {
Some(serde_json::json!({
"name": result.test.name,
"module": result.test.module,
"memory": result.cost.mem,
"cpu": result.cost.cpu
}))
} else {
None
}
})
.collect();
let json = serde_json::json!({
"benchmarks": benchmark_results,
"seed": seed,
});
println!("{}", serde_json::to_string_pretty(&json).unwrap());
}
_ => super::Terminal.handle_event(event),
}
}

21
crates/aiken-project/src/telemetry/terminal.rs
View File

@ -224,14 +224,19 @@ impl EventListener for Terminal {
"...".if_supports_color(Stderr, |s| s.bold())
);
}
Event::FinishedBenchmarks { seed: _, tests: _ } => {
eprintln!(
"{} {}",
" Complete"
.if_supports_color(Stderr, |s| s.bold())
.if_supports_color(Stderr, |s| s.green()),
"benchmark results written to CSV".if_supports_color(Stderr, |s| s.bold())
);
Event::FinishedBenchmarks { tests, .. } => {
for test in tests {
if let TestResult::Benchmark(result) = test {
println!(
"{} {} ",
result.test.name.bold(),
"BENCH".blue(),
);
println!(" Memory: {} bytes", result.cost.mem);
println!(" CPU: {} units", result.cost.cpu);
println!();
}
}
}
}
}

1
crates/aiken-project/src/test_framework.rs
View File

@ -289,7 +289,6 @@ mod test {
result.labels
)
}
// todo riley - should this be reachable?
TestResult::Benchmark(..) => unreachable!("property returned benchmark result ?!"),
}
}

6
crates/aiken/src/cmd/benchmark.rs
View File

@ -34,10 +34,6 @@ pub struct Args {
/// Environment to use for benchmarking
env: Option<String>,
/// Output file for benchmark results
#[clap(short, long)]
output: PathBuf,
}
pub fn exec(
@ -48,7 +44,6 @@ pub fn exec(
seed,
times_to_run,
env,
output,
}: Args,
) -> miette::Result<()> {
let mut rng = rand::thread_rng();
@ -67,7 +62,6 @@ pub fn exec(
seed,
times_to_run,
env.clone(),
output.clone(),
)
},
);

2
crates/aiken/src/cmd/mod.rs
View File

@ -36,7 +36,7 @@ pub enum Cmd {
Docs(docs::Args),
Add(packages::add::Args),
Benchmark(benchmark::Args),
Bench(benchmark::Args),
#[clap(subcommand)]
Blueprint(blueprint::Cmd),

2
crates/aiken/src/main.rs
View File

@ -24,7 +24,7 @@ fn main() -> miette::Result<()> {
Cmd::Build(args) => build::exec(args),
Cmd::Address(args) => address::exec(args),
Cmd::Check(args) => check::exec(args),
Cmd::Benchmark(args) => benchmark::exec(args),
Cmd::Bench(args) => benchmark::exec(args),
Cmd::Docs(args) => docs::exec(args),
Cmd::Add(args) => add::exec(args),
Cmd::Blueprint(args) => blueprint::exec(args),