waalge
/
aiken
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Handle fuzzer failing unexpected 

We shouldn't panic here but bubble the error up to the user to inform
  them about a possibly ill-formed fuzzer.

  Fixes #864.
This commit is contained in:
KtorZ 2024-03-11 01:04:46 +01:00
parent 0e0bed3c9d
commit 4fbb4fe2db
No known key found for this signature in database
GPG Key ID: 33173CB6F77F4277
2 changed files with 120 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
crates/aiken-project/src/telemetry.rs
View File

@ -307,7 +307,15 @@ fn fmt_test(
TestResult::PropertyTestResult(PropertyTestResult { iterations, .. }) => { TestResult::PropertyTestResult(PropertyTestResult { iterations, .. }) => {
test = format!( test = format!(
"{test} [after {} test{}]", "{test} [after {} test{}]",
pretty::pad_left(iterations.to_string(), max_iter, " "), pretty::pad_left(
if *iterations == 0 {
"?".to_string()
} else {
iterations.to_string()
},
max_iter,
" "
),
if *iterations > 1 { "s" } else { "" } if *iterations > 1 { "s" } else { "" }
); );
} }
@ -337,11 +345,19 @@ fn fmt_test(
} }
// CounterExamples // CounterExamples
if let TestResult::PropertyTestResult(PropertyTestResult { if let TestResult::PropertyTestResult(PropertyTestResult { counterexample, .. }) = result {
counterexample: None, match counterexample {
.. Err(err) => {
}) = result test = format!(
{ "{test}\n{}\n{}",
"× fuzzer failed unexpectedly"
.if_supports_color(Stderr, |s| s.red())
.if_supports_color(Stderr, |s| s.bold()),
format!("| {err}").if_supports_color(Stderr, |s| s.red())
);
}
Ok(None) => {
if !result.is_success() { if !result.is_success() {
test = format!( test = format!(
"{test}\n{}", "{test}\n{}",
@ -352,11 +368,7 @@ fn fmt_test(
} }
} }
if let TestResult::PropertyTestResult(PropertyTestResult { Ok(Some(counterexample)) => {
counterexample: Some(counterexample),
..
}) = result
{
let is_expected_failure = result.is_success(); let is_expected_failure = result.is_success();
test = format!( test = format!(
@ -391,6 +403,8 @@ fn fmt_test(
.join("\n"), .join("\n"),
); );
} }
}
}
// Labels // Labels
if let TestResult::PropertyTestResult(PropertyTestResult { labels, .. }) = result { if let TestResult::PropertyTestResult(PropertyTestResult { labels, .. }) = result {

73
crates/aiken-project/src/test_framework.rs
View File

@ -212,6 +212,13 @@ pub struct Fuzzer<T> {
pub stripped_type_info: Rc<Type>, pub stripped_type_info: Rc<Type>,
} }
#[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)]
#[error("Fuzzer exited unexpectedly: {uplc_error}")]
pub struct FuzzerError {
traces: Vec<String>,
uplc_error: uplc::machine::Error,
}
impl PropertyTest { impl PropertyTest {
pub const DEFAULT_MAX_SUCCESS: usize = 100; pub const DEFAULT_MAX_SUCCESS: usize = 100;
@ -222,16 +229,24 @@ impl PropertyTest {
let (traces, counterexample, iterations) = let (traces, counterexample, iterations) =
match self.run_n_times(n, Prng::from_seed(seed), None, &mut labels) { match self.run_n_times(n, Prng::from_seed(seed), None, &mut labels) {
None => (Vec::new(), None, n), Ok(None) => (Vec::new(), Ok(None), n),
Some((remaining, counterexample)) => ( Ok(Some((remaining, counterexample))) => (
self.eval(&counterexample.value) self.eval(&counterexample.value)
.logs() .logs()
.into_iter() .into_iter()
.filter(|s| PropertyTest::extract_label(s).is_none()) .filter(|s| PropertyTest::extract_label(s).is_none())
.collect(), .collect(),
Some(counterexample.value), Ok(Some(counterexample.value)),
n - remaining + 1, n - remaining + 1,
), ),
Err(FuzzerError { traces, uplc_error }) => (
traces
.into_iter()
.filter(|s| PropertyTest::extract_label(s).is_none())
.collect(),
Err(uplc_error),
0,
),
}; };
TestResult::PropertyTestResult(PropertyTestResult { TestResult::PropertyTestResult(PropertyTestResult {
@ -249,11 +264,11 @@ impl PropertyTest {
prng: Prng, prng: Prng,
counterexample: Option<(usize, Counterexample<'a>)>, counterexample: Option<(usize, Counterexample<'a>)>,
labels: &mut BTreeMap<String, usize>, labels: &mut BTreeMap<String, usize>,
) -> Option<(usize, Counterexample<'a>)> { ) -> Result<Option<(usize, Counterexample<'a>)>, FuzzerError> {
// We short-circuit failures in case we have any. The counterexample is already simplified // We short-circuit failures in case we have any. The counterexample is already simplified
// at this point. // at this point.
if remaining > 0 && counterexample.is_none() { if remaining > 0 && counterexample.is_none() {
let (next_prng, counterexample) = self.run_once(prng, labels); let (next_prng, counterexample) = self.run_once(prng, labels)?;
self.run_n_times( self.run_n_times(
remaining - 1, remaining - 1,
next_prng, next_prng,
@ -261,7 +276,7 @@ impl PropertyTest {
labels, labels,
) )
} else { } else {
counterexample Ok(counterexample)
} }
} }
@ -269,10 +284,10 @@ impl PropertyTest {
&self, &self,
prng: Prng, prng: Prng,
labels: &mut BTreeMap<String, usize>, labels: &mut BTreeMap<String, usize>,
) -> (Prng, Option<Counterexample<'_>>) { ) -> Result<(Prng, Option<Counterexample<'_>>), FuzzerError> {
let (next_prng, value) = prng let (next_prng, value) = prng
.sample(&self.fuzzer.program) .sample(&self.fuzzer.program)?
.expect("running seeded Prng cannot fail."); .expect("A seeded PRNG returned 'None' which indicates a fuzzer is ill-formed and implemented wrongly; please contact library's authors.");
let mut result = self.eval(&value); let mut result = self.eval(&value);
@ -297,8 +312,9 @@ impl PropertyTest {
choices: next_prng.choices(), choices: next_prng.choices(),
cache: Cache::new(|choices| { cache: Cache::new(|choices| {
match Prng::from_choices(choices).sample(&self.fuzzer.program) { match Prng::from_choices(choices).sample(&self.fuzzer.program) {
None => Status::Invalid, Err(..) => Status::Invalid,
Some((_, value)) => { Ok(None) => Status::Invalid,
Ok(Some((_, value))) => {
let result = self.eval(&value); let result = self.eval(&value);
let is_failure = result.failed(self.can_error); let is_failure = result.failed(self.can_error);
@ -321,9 +337,9 @@ impl PropertyTest {
counterexample.simplify(); counterexample.simplify();
} }
(next_prng, Some(counterexample)) Ok((next_prng, Some(counterexample)))
} else { } else {
(next_prng, None) Ok((next_prng, None))
} }
} }
@ -431,14 +447,19 @@ impl Prng {
} }
/// Generate a pseudo-random value from a fuzzer using the given PRNG. /// Generate a pseudo-random value from a fuzzer using the given PRNG.
pub fn sample(&self, fuzzer: &Program<Name>) -> Option<(Prng, PlutusData)> { pub fn sample(
&self,
fuzzer: &Program<Name>,
) -> Result<Option<(Prng, PlutusData)>, FuzzerError> {
let program = Program::<NamedDeBruijn>::try_from(fuzzer.apply_data(self.uplc())).unwrap(); let program = Program::<NamedDeBruijn>::try_from(fuzzer.apply_data(self.uplc())).unwrap();
Prng::from_result( let mut result = program.eval(ExBudget::max());
program result
.eval(ExBudget::max())
.result() .result()
.expect("Fuzzer crashed?"), .map_err(|uplc_error| FuzzerError {
) traces: result.logs(),
uplc_error,
})
.map(Prng::from_result)
} }
/// Obtain a Prng back from a fuzzer execution. As a reminder, fuzzers have the following /// Obtain a Prng back from a fuzzer execution. As a reminder, fuzzers have the following
@ -812,7 +833,11 @@ impl<U, T> TestResult<U, T> {
match self { match self {
TestResult::UnitTestResult(UnitTestResult { success, .. }) => *success, TestResult::UnitTestResult(UnitTestResult { success, .. }) => *success,
TestResult::PropertyTestResult(PropertyTestResult { TestResult::PropertyTestResult(PropertyTestResult {
counterexample, counterexample: Err(..),
..
}) => false,
TestResult::PropertyTestResult(PropertyTestResult {
counterexample: Ok(counterexample),
test, test,
.. ..
}) => { }) => {
@ -923,7 +948,7 @@ impl UnitTestResult<(Constant, Rc<Type>)> {
#[derive(Debug)] #[derive(Debug)]
pub struct PropertyTestResult<T> { pub struct PropertyTestResult<T> {
pub test: PropertyTest, pub test: PropertyTest,
pub counterexample: Option<T>, pub counterexample: Result<Option<T>, uplc::machine::Error>,
pub iterations: usize, pub iterations: usize,
pub labels: BTreeMap<String, usize>, pub labels: BTreeMap<String, usize>,
pub traces: Vec<String>, pub traces: Vec<String>,
@ -937,9 +962,11 @@ impl PropertyTestResult<PlutusData> {
data_types: &IndexMap<&DataTypeKey, &TypedDataType>, data_types: &IndexMap<&DataTypeKey, &TypedDataType>,
) -> PropertyTestResult<UntypedExpr> { ) -> PropertyTestResult<UntypedExpr> {
PropertyTestResult { PropertyTestResult {
counterexample: self.counterexample.map(|counterexample| { counterexample: self.counterexample.map(|ok| {
ok.map(|counterexample| {
UntypedExpr::reify_data(data_types, counterexample, &self.test.fuzzer.type_info) UntypedExpr::reify_data(data_types, counterexample, &self.test.fuzzer.type_info)
.expect("Failed to reify counterexample?") .expect("Failed to reify counterexample?")
})
}), }),
iterations: self.iterations, iterations: self.iterations,
test: self.test, test: self.test,
@ -1397,7 +1424,7 @@ mod test {
None, None,
&mut labels, &mut labels,
) { ) {
Some((_, counterexample)) => counterexample, Ok(Some((_, counterexample))) => counterexample,
_ => panic!("expected property to fail but it didn't."), _ => panic!("expected property to fail but it didn't."),
} }
} }