diff --git a/crates/aiken-lang/src/parser.rs b/crates/aiken-lang/src/parser.rs index 7914487d..dc4b04b3 100644 --- a/crates/aiken-lang/src/parser.rs +++ b/crates/aiken-lang/src/parser.rs @@ -24,7 +24,7 @@ pub fn module( ) -> Result<(ast::UntypedModule, ModuleExtra), Vec> { let lexer::LexInfo { tokens, extra } = lexer::run(src)?; - let stream = chumsky::Stream::from_iter(ast::Span::create(tokens.len()), tokens.into_iter()); + let stream = chumsky::Stream::from_iter(ast::Span::create(tokens.len(), 1), tokens.into_iter()); let definitions = definition().repeated().then_ignore(end()).parse(stream)?; diff --git a/crates/aiken-lang/src/parser/utils.rs b/crates/aiken-lang/src/parser/utils.rs index 2e1716ce..0ce3e404 100644 --- a/crates/aiken-lang/src/parser/utils.rs +++ b/crates/aiken-lang/src/parser/utils.rs @@ -101,7 +101,7 @@ macro_rules! assert_expr { let $crate::parser::lexer::LexInfo { tokens, .. } = $crate::parser::lexer::run(indoc::indoc! { $code }).unwrap(); - let stream = chumsky::Stream::from_iter($crate::ast::Span::create(tokens.len()), tokens.into_iter()); + let stream = chumsky::Stream::from_iter($crate::ast::Span::create(tokens.len(), 1), tokens.into_iter()); let result = $crate::parser::expr::sequence().parse(stream).unwrap(); @@ -122,7 +122,7 @@ macro_rules! assert_annotation { let $crate::parser::lexer::LexInfo { tokens, .. } = $crate::parser::lexer::run(indoc::indoc! { $code }).unwrap(); - let stream = chumsky::Stream::from_iter($crate::ast::Span::create(tokens.len()), tokens.into_iter()); + let stream = chumsky::Stream::from_iter($crate::ast::Span::create(tokens.len(), 1), tokens.into_iter()); let result = $crate::parser::annotation().parse(stream).unwrap(); @@ -159,7 +159,7 @@ macro_rules! assert_definition { let $crate::parser::lexer::LexInfo { tokens, .. } = $crate::parser::lexer::run(indoc::indoc! { $code }).unwrap(); - let stream = chumsky::Stream::from_iter($crate::ast::Span::create(tokens.len()), tokens.into_iter()); + let stream = chumsky::Stream::from_iter($crate::ast::Span::create(tokens.len(), 1), tokens.into_iter()); let result = $crate::parser::definition().parse(stream).unwrap(); diff --git a/crates/aiken-lang/src/tests/snapshots/parse_unicode_offset_1.snap b/crates/aiken-lang/src/tests/snapshots/parse_unicode_offset_1.snap new file mode 100644 index 00000000..fd21a8e8 --- /dev/null +++ b/crates/aiken-lang/src/tests/snapshots/parse_unicode_offset_1.snap @@ -0,0 +1,52 @@ +--- +source: crates/aiken-lang/src/tests/parser.rs +description: "Code:\n\nfn foo() {\n let x = \"★\"\n x\n}\n" +--- +Module { + name: "", + docs: [], + type_info: (), + definitions: [ + Fn( + Function { + arguments: [], + body: Sequence { + location: 13..30, + expressions: [ + Assignment { + location: 13..26, + value: ByteArray { + location: 21..26, + bytes: [ + 226, + 152, + 133, + ], + preferred_format: Utf8String, + }, + pattern: Var { + location: 17..18, + name: "x", + }, + kind: Let, + annotation: None, + }, + Var { + location: 29..30, + name: "x", + }, + ], + }, + doc: None, + location: 0..8, + name: "foo", + public: false, + return_annotation: None, + return_type: (), + end_position: 31, + can_error: true, + }, + ), + ], + kind: Validator, +} diff --git a/crates/aiken-lang/src/tests/snapshots/parse_unicode_offset_2.snap b/crates/aiken-lang/src/tests/snapshots/parse_unicode_offset_2.snap new file mode 100644 index 00000000..f0186e94 --- /dev/null +++ b/crates/aiken-lang/src/tests/snapshots/parse_unicode_offset_2.snap @@ -0,0 +1,50 @@ +--- +source: crates/aiken-lang/src/tests/parser.rs +description: "Code:\n\nfn foo() {\n let x = \"*\"\n x\n}\n" +--- +Module { + name: "", + docs: [], + type_info: (), + definitions: [ + Fn( + Function { + arguments: [], + body: Sequence { + location: 13..28, + expressions: [ + Assignment { + location: 13..24, + value: ByteArray { + location: 21..24, + bytes: [ + 42, + ], + preferred_format: Utf8String, + }, + pattern: Var { + location: 17..18, + name: "x", + }, + kind: Let, + annotation: None, + }, + Var { + location: 27..28, + name: "x", + }, + ], + }, + doc: None, + location: 0..8, + name: "foo", + public: false, + return_annotation: None, + return_type: (), + end_position: 29, + can_error: true, + }, + ), + ], + kind: Validator, +}