Merge branch 'parser-newline-comment-fix'

crates/aiken-lang/src/parser/lexer.rs

@@ -55,8 +55,6 @@ pub fn lexer() -> impl Parser<char, Vec<(Token, Span)>, Error = ParseError> {
         just('|').to(Token::Vbar),
         just("&&").to(Token::AmperAmper),
         just('#').to(Token::Hash),
-        choice((just("\n\n"), just("\r\n\r\n"))).to(Token::EmptyLine),
-        choice((just("\n"), just("\r\n"))).to(Token::NewLine),
     ));
 
     let grouping = choice((
@@ -126,29 +124,41 @@ pub fn lexer() -> impl Parser<char, Vec<(Token, Span)>, Error = ParseError> {
         }
     });
 
-    let module_comments = just("////").ignore_then(
-        take_until(text::newline().rewind())
-            .to(Token::ModuleComment)
-            .map_with_span(|token, span| (token, span)),
-    );
-
-    let doc_comments = just("///").ignore_then(
-        take_until(text::newline().rewind())
-            .to(Token::DocComment)
-            .map_with_span(|token, span| (token, span)),
-    );
-
-    let comments = just("//").ignore_then(
-        take_until(text::newline().rewind())
-            .to(Token::Comment)
-            .map_with_span(|token, span| (token, span)),
-    );
+    fn comment_parser(token: Token) -> impl Parser<char, (Token, Span), Error = ParseError> {
+        let n = match token {
+            Token::ModuleComment => 4,
+            Token::DocComment => 3,
+            Token::Comment => 2,
+            _ => unreachable!(),
+        };
 
         choice((
-        module_comments,
-        doc_comments,
-        comments,
-        choice((ordinal, keyword, int, op, grouping, string))
+            // NOTE: The first case here work around a bug introduced with chumsky=0.9.0 which
+            // miscalculate the offset for empty comments.
+            just("/".repeat(n))
+                .ignore_then(text::newline().rewind())
+                .to(token.clone())
+                .map_with_span(move |token, span: Span| {
+                    (token, Span::new((), span.start + n..span.end))
+                }),
+            just("/".repeat(n)).ignore_then(
+                take_until(text::newline().rewind())
+                    .to(token)
+                    .map_with_span(|token, span| (token, span)),
+            ),
+        ))
+    }
+
+    let newlines = choice((
+        choice((just("\n\n"), just("\r\n\r\n"))).to(Token::EmptyLine),
+        choice((just("\n"), just("\r\n"))).to(Token::NewLine),
+    ));
+
+    choice((
+        comment_parser(Token::ModuleComment),
+        comment_parser(Token::DocComment),
+        comment_parser(Token::Comment),
+        choice((ordinal, keyword, int, op, newlines, grouping, string))
             .or(any().map(Token::Error).validate(|t, span, emit| {
                 emit(ParseError::expected_input_found(
                     span,

crates/aiken-lang/src/tests/format.rs

@@ -415,3 +415,91 @@ fn test_trace_if_false() {
 
     assert_fmt(src, src);
 }
+
+#[test]
+fn test_newline_comments() {
+    let src = indoc! {r#"
+        // My comment
+        //
+        // has a newline.
+        fn foo() {
+          True
+        }
+
+        // My comments
+
+        // can live apart
+        fn bar() {
+          True
+        }
+    "#};
+
+    assert_fmt(src, src);
+}
+
+#[test]
+fn test_newline_doc_comments() {
+    let src = indoc! {r#"
+        /// My doc comment
+        ///
+        /// has a newline.
+        fn foo() {
+          True
+        }
+
+        /// My doc comments
+
+        /// cannot be separated
+        fn bar() {
+          True
+        }
+    "#};
+
+    let out = indoc! {r#"
+        /// My doc comment
+        ///
+        /// has a newline.
+        fn foo() {
+          True
+        }
+
+        /// My doc comments
+        /// cannot be separated
+        fn bar() {
+          True
+        }
+    "#};
+
+    assert_fmt(src, out);
+}
+
+#[test]
+fn test_newline_module_comments() {
+    let src = indoc! {r#"
+        //// My module comment
+        ////
+        //// has a newline.
+
+        fn foo() {
+          True
+        }
+
+        //// My module comments
+
+        //// cannot be separated
+    "#};
+
+    let out = indoc! {r#"
+        //// My module comment
+        ////
+        //// has a newline.
+        //// My module comments
+        //// cannot be separated
+
+        fn foo() {
+          True
+        }
+    "#};
+
+    assert_fmt(src, out);
+}