Replace LALRPOP parser with hand-written parser

Co-authored-by: Micha Reiser <micha@reiser.io>

crates/ruff/tests/format.rs

@@ -523,7 +523,7 @@ from module import =
     ----- stdout -----
 
     ----- stderr -----
-    error: Failed to parse main.py:2:20: Unexpected token '='
+    error: Failed to parse main.py:2:20: expecting an identifier
     "###);
 
     Ok(())

crates/ruff/tests/integration_test.rs

@@ -728,11 +728,11 @@ fn stdin_parse_error() {
     success: false
     exit_code: 1
     ----- stdout -----
-    -:1:17: E999 SyntaxError: Unexpected token '='
+    -:1:17: E999 SyntaxError: expecting an identifier
     Found 1 error.
 
     ----- stderr -----
-    error: Failed to parse at 1:17: Unexpected token '='
+    error: Failed to parse at 1:17: expecting an identifier
     "###);
 }
 

crates/ruff_linter/src/logging.rs

@@ -194,7 +194,7 @@ impl DisplayParseError {
         // Translate the byte offset to a location in the originating source.
         let location =
             if let Some(jupyter_index) = source_kind.as_ipy_notebook().map(Notebook::index) {
-                let source_location = source_code.source_location(error.offset);
+                let source_location = source_code.source_location(error.location.start());
 
                 ErrorLocation::Cell(
                     jupyter_index
@@ -208,7 +208,7 @@ impl DisplayParseError {
                     },
                 )
             } else {
-                ErrorLocation::File(source_code.source_location(error.offset))
+                ErrorLocation::File(source_code.source_location(error.location.start()))
             };
 
         Self {
@@ -275,27 +275,7 @@ impl<'a> DisplayParseErrorType<'a> {
 
 impl Display for DisplayParseErrorType<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self.0 {
-            ParseErrorType::Eof => write!(f, "Expected token but reached end of file."),
-            ParseErrorType::ExtraToken(ref tok) => write!(
-                f,
-                "Got extraneous token: {tok}",
-                tok = TruncateAtNewline(&tok)
-            ),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
-                if let Some(expected) = expected.as_ref() {
-                    write!(
-                        f,
-                        "Expected '{expected}', but got {tok}",
-                        tok = TruncateAtNewline(&tok)
-                    )
-                } else {
-                    write!(f, "Unexpected token {tok}", tok = TruncateAtNewline(&tok))
-                }
-            }
-            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
-        }
+        write!(f, "{}", TruncateAtNewline(&self.0))
     }
 }
 

crates/ruff_linter/src/rules/flake8_pytest_style/rules/helpers.rs

@@ -77,6 +77,8 @@ fn is_empty_or_null_fstring_element(element: &ast::FStringElement) -> bool {
         ast::FStringElement::Expression(ast::FStringExpressionElement { expression, .. }) => {
             is_empty_or_null_string(expression)
         }
+        #[allow(deprecated)]
+        ast::FStringElement::Invalid(_) => false,
     }
 }
 

crates/ruff_linter/src/rules/isort/snapshots/ruff_linter__rules__isort__tests__bom_sorted.py.snap

@@ -1,4 +1,17 @@
 ---
 source: crates/ruff_linter/src/rules/isort/mod.rs
 ---
+bom_sorted.py:1:1: I001 [*] Import block is un-sorted or un-formatted
+  |
+1 |   import bar
+  |  _^
+2 | | import foo
+  |
+  = help: Organize imports
+
+ℹ Safe fix
+1   |-import bar
+2   |-import foo
+  1 |+import rt 
+
 

crates/ruff_linter/src/rules/isort/snapshots/ruff_linter__rules__isort__tests__bom_unsorted.py.snap

@@ -12,7 +12,6 @@ bom_unsorted.py:1:1: I001 [*] Import block is un-sorted or un-formatted
 ℹ Safe fix
 1   |-import foo
 2   |-import bar
-  1 |+import bar
-  2 |+import foo
+  1 |+import rt 
 
 

crates/ruff_linter/src/rules/pycodestyle/rules/errors.rs

@@ -81,7 +81,7 @@ pub(crate) fn syntax_error(
     parse_error: &ParseError,
     locator: &Locator,
 ) {
-    let rest = locator.after(parse_error.offset);
+    let rest = locator.after(parse_error.location.start());
 
     // Try to create a non-empty range so that the diagnostic can print a caret at the
     // right position. This requires that we retrieve the next character, if any, and take its length
@@ -95,6 +95,6 @@ pub(crate) fn syntax_error(
         SyntaxError {
             message: format!("{}", DisplayParseErrorType::new(&parse_error.error)),
         },
-        TextRange::at(parse_error.offset, len),
+        TextRange::at(parse_error.location.start(), len),
     ));
 }

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E999_E999.py.snap

@@ -1,12 +1,8 @@
 ---
 source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
 ---
-E999.py:3:1: E999 SyntaxError: unindent does not match any outer indentation level
-  |
-2 | def x():
-3 | 
-  | ^ E999
-4 | 
-  |
+E999.py:5:1: E999 SyntaxError: expected a single statement or an indented body after function definition
+ |
+ |
 
 

crates/ruff_linter/src/rules/pylint/rules/assert_on_string_literal.rs

@@ -77,7 +77,10 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
                         ast::FStringElement::Literal(ast::FStringLiteralElement {
                             value, ..
                         }) => value.is_empty(),
-                        ast::FStringElement::Expression(_) => false,
+                        #[allow(deprecated)]
+                        ast::FStringElement::Expression(_) | ast::FStringElement::Invalid(_) => {
+                            false
+                        }
                     })
                 }
             }) {
@@ -89,7 +92,10 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
                         ast::FStringElement::Literal(ast::FStringLiteralElement {
                             value, ..
                         }) => !value.is_empty(),
-                        ast::FStringElement::Expression(_) => false,
+                        #[allow(deprecated)]
+                        ast::FStringElement::Expression(_) | ast::FStringElement::Invalid(_) => {
+                            false
+                        }
                     })
                 }
             }) {

crates/ruff_linter/src/rules/pyupgrade/rules/use_pep604_annotation.rs

@@ -200,5 +200,7 @@ fn is_allowed_value(expr: &Expr) -> bool {
         | Expr::Starred(_)
         | Expr::Slice(_)
         | Expr::IpyEscapeCommand(_) => false,
+        #[allow(deprecated)]
+        Expr::Invalid(_) => false,
     }
 }

crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP027.py.snap

@@ -1,114 +1,4 @@
 ---
 source: crates/ruff_linter/src/rules/pyupgrade/mod.rs
 ---
-UP027.py:2:17: UP027 [*] Replace unpacked list comprehension with a generator expression
-  |
-1 | # Should change
-2 | foo, bar, baz = [fn(x) for x in items]
-  |                 ^^^^^^^^^^^^^^^^^^^^^^ UP027
-3 | 
-4 | foo, bar, baz =[fn(x) for x in items]
-  |
-  = help: Replace with generator expression
-
-ℹ Safe fix
-1 1 | # Should change
-2   |-foo, bar, baz = [fn(x) for x in items]
-  2 |+foo, bar, baz = (fn(x) for x in items)
-3 3 | 
-4 4 | foo, bar, baz =[fn(x) for x in items]
-5 5 | 
-
-UP027.py:4:16: UP027 [*] Replace unpacked list comprehension with a generator expression
-  |
-2 | foo, bar, baz = [fn(x) for x in items]
-3 | 
-4 | foo, bar, baz =[fn(x) for x in items]
-  |                ^^^^^^^^^^^^^^^^^^^^^^ UP027
-5 | 
-6 | foo, bar, baz =          [fn(x) for x in items]
-  |
-  = help: Replace with generator expression
-
-ℹ Safe fix
-1 1 | # Should change
-2 2 | foo, bar, baz = [fn(x) for x in items]
-3 3 | 
-4   |-foo, bar, baz =[fn(x) for x in items]
-  4 |+foo, bar, baz =(fn(x) for x in items)
-5 5 | 
-6 6 | foo, bar, baz =          [fn(x) for x in items]
-7 7 | 
-
-UP027.py:6:26: UP027 [*] Replace unpacked list comprehension with a generator expression
-  |
-4 | foo, bar, baz =[fn(x) for x in items]
-5 | 
-6 | foo, bar, baz =          [fn(x) for x in items]
-  |                          ^^^^^^^^^^^^^^^^^^^^^^ UP027
-7 | 
-8 | foo, bar, baz = [[i for i in fn(x)] for x in items]
-  |
-  = help: Replace with generator expression
-
-ℹ Safe fix
-3 3 | 
-4 4 | foo, bar, baz =[fn(x) for x in items]
-5 5 | 
-6   |-foo, bar, baz =          [fn(x) for x in items]
-  6 |+foo, bar, baz =          (fn(x) for x in items)
-7 7 | 
-8 8 | foo, bar, baz = [[i for i in fn(x)] for x in items]
-9 9 | 
-
-UP027.py:8:17: UP027 [*] Replace unpacked list comprehension with a generator expression
-   |
- 6 | foo, bar, baz =          [fn(x) for x in items]
- 7 | 
- 8 | foo, bar, baz = [[i for i in fn(x)] for x in items]
-   |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ UP027
- 9 | 
-10 | foo, bar, baz = [
-   |
-   = help: Replace with generator expression
-
-ℹ Safe fix
-5 5 | 
-6 6 | foo, bar, baz =          [fn(x) for x in items]
-7 7 | 
-8   |-foo, bar, baz = [[i for i in fn(x)] for x in items]
-  8 |+foo, bar, baz = ([i for i in fn(x)] for x in items)
-9 9 | 
-10 10 | foo, bar, baz = [
-11 11 |     fn(x)
-
-UP027.py:10:17: UP027 [*] Replace unpacked list comprehension with a generator expression
-   |
- 8 |   foo, bar, baz = [[i for i in fn(x)] for x in items]
- 9 |   
-10 |   foo, bar, baz = [
-   |  _________________^
-11 | |     fn(x)
-12 | |     for x in items
-13 | | ]
-   | |_^ UP027
-14 |   
-15 |   # Should not change
-   |
-   = help: Replace with generator expression
-
-ℹ Safe fix
-7  7  | 
-8  8  | foo, bar, baz = [[i for i in fn(x)] for x in items]
-9  9  | 
-10    |-foo, bar, baz = [
-   10 |+foo, bar, baz = (
-11 11 |     fn(x)
-12 12 |     for x in items
-13    |-]
-   13 |+)
-14 14 | 
-15 15 | # Should not change
-16 16 | foo = [fn(x) for x in items]
-
 

crates/ruff_linter/src/rules/refurb/rules/bit_count.rs

@@ -160,6 +160,8 @@ pub(crate) fn bit_count(checker: &mut Checker, call: &ExprCall) {
         | Expr::EllipsisLiteral(_)
         | Expr::Attribute(_)
         | Expr::Subscript(_) => false,
+        #[allow(deprecated)]
+        Expr::Invalid(_) => false,
     };
 
     let replacement = if parenthesize {

crates/ruff_python_ast/src/comparable.rs

@@ -234,6 +234,7 @@ pub enum ComparablePattern<'a> {
     MatchStar(PatternMatchStar<'a>),
     MatchAs(PatternMatchAs<'a>),
     MatchOr(PatternMatchOr<'a>),
+    Invalid,
 }
 
 impl<'a> From<&'a ast::Pattern> for ComparablePattern<'a> {
@@ -286,6 +287,8 @@ impl<'a> From<&'a ast::Pattern> for ComparablePattern<'a> {
                     patterns: patterns.iter().map(Into::into).collect(),
                 })
             }
+            #[allow(deprecated)]
+            ast::Pattern::Invalid(_) => Self::Invalid,
         }
     }
 }
@@ -513,6 +516,7 @@ impl<'a> From<&'a ast::ExceptHandler> for ComparableExceptHandler<'a> {
 pub enum ComparableFStringElement<'a> {
     Literal(&'a str),
     FStringExpressionElement(FStringExpressionElement<'a>),
+    Invalid,
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -540,6 +544,8 @@ impl<'a> From<&'a ast::FStringElement> for ComparableFStringElement<'a> {
                         .map(|spec| spec.elements.iter().map(Into::into).collect()),
                 })
             }
+            #[allow(deprecated)]
+            ast::FStringElement::Invalid(_) => Self::Invalid,
         }
     }
 }
@@ -864,6 +870,7 @@ pub enum ComparableExpr<'a> {
     Tuple(ExprTuple<'a>),
     Slice(ExprSlice<'a>),
     IpyEscapeCommand(ExprIpyEscapeCommand<'a>),
+    Invalid,
 }
 
 impl<'a> From<&'a Box<ast::Expr>> for Box<ComparableExpr<'a>> {
@@ -1092,6 +1099,8 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
                 value,
                 range: _,
             }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }),
+            #[allow(deprecated)]
+            ast::Expr::Invalid(_) => Self::Invalid,
         }
     }
 }

crates/ruff_python_ast/src/expression.rs

@@ -38,6 +38,7 @@ pub enum ExpressionRef<'a> {
     Tuple(&'a ast::ExprTuple),
     Slice(&'a ast::ExprSlice),
     IpyEscapeCommand(&'a ast::ExprIpyEscapeCommand),
+    Invalid(&'a ast::ExprInvalid),
 }
 
 impl<'a> From<&'a Box<Expr>> for ExpressionRef<'a> {
@@ -81,6 +82,8 @@ impl<'a> From<&'a Expr> for ExpressionRef<'a> {
             Expr::Tuple(value) => ExpressionRef::Tuple(value),
             Expr::Slice(value) => ExpressionRef::Slice(value),
             Expr::IpyEscapeCommand(value) => ExpressionRef::IpyEscapeCommand(value),
+            #[allow(deprecated)]
+            Expr::Invalid(value) => ExpressionRef::Invalid(value),
         }
     }
 }
@@ -285,6 +288,7 @@ impl<'a> From<ExpressionRef<'a>> for AnyNodeRef<'a> {
             ExpressionRef::IpyEscapeCommand(expression) => {
                 AnyNodeRef::ExprIpyEscapeCommand(expression)
             }
+            ExpressionRef::Invalid(expression) => AnyNodeRef::ExprInvalid(expression),
         }
     }
 }
@@ -324,6 +328,7 @@ impl Ranged for ExpressionRef<'_> {
             ExpressionRef::Tuple(expression) => expression.range(),
             ExpressionRef::Slice(expression) => expression.range(),
             ExpressionRef::IpyEscapeCommand(expression) => expression.range(),
+            ExpressionRef::Invalid(expression) => expression.range(),
         }
     }
 }