Replace LALRPOP parser with hand-written parser

crates/ruff/tests/format.rs

@@ -521,7 +521,7 @@ from module import =
     ----- stdout -----
 
     ----- stderr -----
-    error: Failed to parse main.py:2:20: Unexpected token '='
+    error: Failed to parse main.py:2:20: expecting an identifier
     "###);
 
     Ok(())

crates/ruff/tests/integration_test.rs

@@ -727,11 +727,11 @@ fn stdin_parse_error() {
     success: false
     exit_code: 1
     ----- stdout -----
-    -:1:17: E999 SyntaxError: Unexpected token '='
+    -:1:17: E999 SyntaxError: expecting an identifier
     Found 1 error.
 
     ----- stderr -----
-    error: Failed to parse at 1:17: Unexpected token '='
+    error: Failed to parse at 1:17: expecting an identifier
     "###);
 }
 

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -134,6 +134,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
             elts,
             ctx,
             range: _,
+            parenthesized: _,
         })
         | Expr::List(ast::ExprList {
             elts,

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -1298,6 +1298,7 @@ where
                                 elts,
                                 ctx,
                                 range: _,
+                                parenthesized: _,
                             }) = slice.as_ref()
                             {
                                 let mut iter = elts.iter();

crates/ruff_linter/src/logging.rs

@@ -194,7 +194,7 @@ impl DisplayParseError {
         // Translate the byte offset to a location in the originating source.
         let location =
             if let Some(jupyter_index) = source_kind.as_ipy_notebook().map(Notebook::index) {
-                let source_location = source_code.source_location(error.offset);
+                let source_location = source_code.source_location(error.location.start());
 
                 ErrorLocation::Cell(
                     jupyter_index
@@ -208,7 +208,7 @@ impl DisplayParseError {
                     },
                 )
             } else {
-                ErrorLocation::File(source_code.source_location(error.offset))
+                ErrorLocation::File(source_code.source_location(error.location.start()))
             };
 
         Self {
@@ -275,27 +275,7 @@ impl<'a> DisplayParseErrorType<'a> {
 
 impl Display for DisplayParseErrorType<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self.0 {
-            ParseErrorType::Eof => write!(f, "Expected token but reached end of file."),
-            ParseErrorType::ExtraToken(ref tok) => write!(
-                f,
-                "Got extraneous token: {tok}",
-                tok = TruncateAtNewline(&tok)
-            ),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
-                if let Some(expected) = expected.as_ref() {
-                    write!(
-                        f,
-                        "Expected '{expected}', but got {tok}",
-                        tok = TruncateAtNewline(&tok)
-                    )
-                } else {
-                    write!(f, "Unexpected token {tok}", tok = TruncateAtNewline(&tok))
-                }
-            }
-            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
-        }
+        write!(f, "{}", TruncateAtNewline(&self.0))
     }
 }
 

crates/ruff_linter/src/rules/flake8_bugbear/rules/duplicate_exceptions.rs

@@ -109,6 +109,7 @@ fn type_pattern(elts: Vec<&Expr>) -> Expr {
         elts: elts.into_iter().cloned().collect(),
         ctx: ExprContext::Load,
         range: TextRange::default(),
+        parenthesized: true,
     }
     .into()
 }

crates/ruff_linter/src/rules/flake8_pie/rules/multiple_starts_ends_with.rs

@@ -173,6 +173,7 @@ pub(crate) fn multiple_starts_ends_with(checker: &mut Checker, expr: &Expr) {
                     .collect(),
                 ctx: ExprContext::Load,
                 range: TextRange::default(),
+                parenthesized: true,
             });
             let node1 = Expr::Name(ast::ExprName {
                 id: arg_name.into(),

crates/ruff_linter/src/rules/flake8_pyi/rules/unnecessary_literal_union.rs

@@ -72,6 +72,7 @@ pub(crate) fn unnecessary_literal_union<'a>(checker: &mut Checker, expr: &'a Exp
                     elts,
                     range: _,
                     ctx: _,
+                    parenthesized: _,
                 }) = slice.as_ref()
                 {
                     for expr in elts {
@@ -123,6 +124,7 @@ pub(crate) fn unnecessary_literal_union<'a>(checker: &mut Checker, expr: &'a Exp
                 elts: literal_exprs.into_iter().cloned().collect(),
                 range: TextRange::default(),
                 ctx: ExprContext::Load,
+                parenthesized: true,
             })),
             range: TextRange::default(),
             ctx: ExprContext::Load,
@@ -148,6 +150,7 @@ pub(crate) fn unnecessary_literal_union<'a>(checker: &mut Checker, expr: &'a Exp
                             elts,
                             range: TextRange::default(),
                             ctx: ExprContext::Load,
+                            parenthesized: true,
                         })),
                         range: TextRange::default(),
                         ctx: ExprContext::Load,

crates/ruff_linter/src/rules/flake8_pyi/rules/unnecessary_type_union.rs

@@ -130,6 +130,7 @@ pub(crate) fn unnecessary_type_union<'a>(checker: &mut Checker, union: &'a Expr)
                                 .collect(),
                             ctx: ExprContext::Load,
                             range: TextRange::default(),
+                            parenthesized: true,
                         })),
                         ctx: ExprContext::Load,
                         range: TextRange::default(),
@@ -151,6 +152,7 @@ pub(crate) fn unnecessary_type_union<'a>(checker: &mut Checker, union: &'a Expr)
                             elts: exprs.into_iter().cloned().collect(),
                             ctx: ExprContext::Load,
                             range: TextRange::default(),
+                            parenthesized: true,
                         })),
                         ctx: ExprContext::Load,
                         range: TextRange::default(),

crates/ruff_linter/src/rules/flake8_pytest_style/rules/helpers.rs

@@ -74,6 +74,8 @@ fn is_empty_or_null_fstring_element(element: &ast::FStringElement) -> bool {
         ast::FStringElement::Expression(ast::FStringExpressionElement { expression, .. }) => {
             is_empty_or_null_string(expression)
         }
+        #[allow(deprecated)]
+        ast::FStringElement::Invalid(_) => false,
     }
 }
 

crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs

@@ -337,6 +337,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
                                 .collect(),
                             ctx: ExprContext::Load,
                             range: TextRange::default(),
+                            parenthesized: true,
                         });
                         diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement(
                             format!("({})", checker.generator().expr(&node)),
@@ -444,6 +445,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
                             elts: elts.clone(),
                             ctx: ExprContext::Load,
                             range: TextRange::default(),
+                            parenthesized: true,
                         });
                         diagnostic.set_fix(Fix::unsafe_edit(Edit::range_replacement(
                             format!("({})", checker.generator().expr(&node)),

crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs

@@ -428,6 +428,7 @@ pub(crate) fn duplicate_isinstance_call(checker: &mut Checker, expr: &Expr) {
                         .collect(),
                     ctx: ExprContext::Load,
                     range: TextRange::default(),
+                    parenthesized: true,
                 };
                 let node1 = ast::ExprName {
                     id: "isinstance".into(),
@@ -543,6 +544,7 @@ pub(crate) fn compare_with_tuple(checker: &mut Checker, expr: &Expr) {
             elts: comparators.into_iter().map(Clone::clone).collect(),
             ctx: ExprContext::Load,
             range: TextRange::default(),
+            parenthesized: true,
         };
         let node1 = ast::ExprName {
             id: id.into(),
@@ -718,7 +720,7 @@ fn get_short_circuit_edit(
         generator.expr(expr)
     };
     Edit::range_replacement(
-        if matches!(expr, Expr::Tuple(ast::ExprTuple { elts, ctx: _, range: _}) if !elts.is_empty())
+        if matches!(expr, Expr::Tuple(ast::ExprTuple { elts, ctx: _, range: _, parenthesized: _}) if !elts.is_empty())
         {
             format!("({content})")
         } else {

crates/ruff_linter/src/rules/isort/snapshots/ruff_linter__rules__isort__tests__bom_sorted.py.snap

@@ -1,4 +1,17 @@
 ---
 source: crates/ruff_linter/src/rules/isort/mod.rs
 ---
+bom_sorted.py:1:1: I001 [*] Import block is un-sorted or un-formatted
+  |
+1 |   import bar
+  |  _^
+2 | | import foo
+  |
+  = help: Organize imports
+
+ℹ Safe fix
+1   |-import bar
+2   |-import foo
+  1 |+import rt 
+
 

crates/ruff_linter/src/rules/isort/snapshots/ruff_linter__rules__isort__tests__bom_unsorted.py.snap

@@ -12,7 +12,6 @@ bom_unsorted.py:1:1: I001 [*] Import block is un-sorted or un-formatted
 ℹ Safe fix
 1   |-import foo
 2   |-import bar
-  1 |+import bar
-  2 |+import foo
+  1 |+import rt 
 
 

crates/ruff_linter/src/rules/pycodestyle/rules/errors.rs

@@ -81,7 +81,7 @@ pub(crate) fn syntax_error(
     parse_error: &ParseError,
     locator: &Locator,
 ) {
-    let rest = locator.after(parse_error.offset);
+    let rest = locator.after(parse_error.location.start());
 
     // Try to create a non-empty range so that the diagnostic can print a caret at the
     // right position. This requires that we retrieve the next character, if any, and take its length
@@ -95,6 +95,6 @@ pub(crate) fn syntax_error(
         SyntaxError {
             message: format!("{}", DisplayParseErrorType::new(&parse_error.error)),
         },
-        TextRange::at(parse_error.offset, len),
+        TextRange::at(parse_error.location.start(), len),
     ));
 }

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E999_E999.py.snap

@@ -1,12 +1,8 @@
 ---
 source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
 ---
-E999.py:3:1: E999 SyntaxError: unindent does not match any outer indentation level
-  |
-2 | def x():
-3 | 
-  | ^ E999
-4 | 
-  |
+E999.py:5:1: E999 SyntaxError: expected a single statement or an indented body after function definition
+ |
+ |
 
 

crates/ruff_linter/src/rules/pylint/rules/assert_on_string_literal.rs

@@ -77,7 +77,10 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
                         ast::FStringElement::Literal(ast::FStringLiteralElement {
                             value, ..
                         }) => value.is_empty(),
-                        ast::FStringElement::Expression(_) => false,
+                        #[allow(deprecated)]
+                        ast::FStringElement::Expression(_) | ast::FStringElement::Invalid(_) => {
+                            false
+                        }
                     })
                 }
             }) {
@@ -89,7 +92,10 @@ pub(crate) fn assert_on_string_literal(checker: &mut Checker, test: &Expr) {
                         ast::FStringElement::Literal(ast::FStringLiteralElement {
                             value, ..
                         }) => !value.is_empty(),
-                        ast::FStringElement::Expression(_) => false,
+                        #[allow(deprecated)]
+                        ast::FStringElement::Expression(_) | ast::FStringElement::Invalid(_) => {
+                            false
+                        }
                     })
                 }
             }) {

crates/ruff_linter/src/rules/pylint/rules/redefined_loop_name.rs

@@ -305,6 +305,7 @@ fn assignment_targets_from_expr<'a>(
             ctx: ExprContext::Store,
             elts,
             range: _,
+            parenthesized: _,
         }) => Box::new(
             elts.iter()
                 .flat_map(|elt| assignment_targets_from_expr(elt, dummy_variable_rgx)),

crates/ruff_linter/src/rules/pylint/rules/repeated_equality_comparison.rs

@@ -143,6 +143,7 @@ pub(crate) fn repeated_equality_comparison(checker: &mut Checker, bool_op: &ast:
                         elts: comparators.iter().copied().cloned().collect(),
                         range: TextRange::default(),
                         ctx: ExprContext::Load,
+                        parenthesized: true,
                     })]),
                     range: bool_op.range(),
                 })),

crates/ruff_linter/src/rules/pyupgrade/rules/os_error_alias.rs

@@ -127,6 +127,7 @@ fn tuple_diagnostic(checker: &mut Checker, tuple: &ast::ExprTuple, aliases: &[&E
                 elts: remaining,
                 ctx: ExprContext::Load,
                 range: TextRange::default(),
+                parenthesized: true,
             };
             format!("({})", checker.generator().expr(&node.into()))
         };

crates/ruff_linter/src/rules/pyupgrade/rules/timeout_error_alias.rs

@@ -141,6 +141,7 @@ fn tuple_diagnostic(checker: &mut Checker, tuple: &ast::ExprTuple, aliases: &[&E
                 elts: remaining,
                 ctx: ExprContext::Load,
                 range: TextRange::default(),
+                parenthesized: true,
             };
             format!("({})", checker.generator().expr(&node.into()))
         };

crates/ruff_linter/src/rules/pyupgrade/rules/use_pep604_annotation.rs

@@ -200,5 +200,7 @@ fn is_allowed_value(expr: &Expr) -> bool {
         | Expr::Starred(_)
         | Expr::Slice(_)
         | Expr::IpyEscapeCommand(_) => false,
+        #[allow(deprecated)]
+        Expr::Invalid(_) => false,
     }
 }

crates/ruff_linter/src/rules/pyupgrade/rules/use_pep695_type_alias.rs

@@ -127,6 +127,7 @@ pub(crate) fn non_pep695_type_alias(checker: &mut Checker, stmt: &StmtAnnAssign)
                                     range: TextRange::default(),
                                     elts: constraints.into_iter().cloned().collect(),
                                     ctx: ast::ExprContext::Load,
+                                    parenthesized: true,
                                 })))
                             }
                             None => None,