Skip to content

Commit

Permalink
Emit LexError for dedent to incorrect level
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaReiser committed Sep 25, 2023
1 parent 01843af commit 67b69ed
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@
):
pass

z = (
z = (
a
+
# a: extracts this comment
Expand All @@ -377,7 +377,7 @@
x and y
)
)
)
)

z = (
(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ if (
):
pass
z = (
z = (
a
+
# a: extracts this comment
Expand All @@ -383,7 +383,7 @@ if (
x and y
)
)
)
)
z = (
(
Expand Down
43 changes: 35 additions & 8 deletions crates/ruff_python_parser/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -556,10 +556,22 @@ impl<'source> Lexer<'source> {
pub fn next_token(&mut self) -> LexResult {
// Return dedent tokens until the current indentation level matches the indentation of the next token.
if let Some(indentation) = self.pending_indentation.take() {
if let Ok(Ordering::Greater) = self.indentations.current().try_compare(indentation) {
self.pending_indentation = Some(indentation);
self.indentations.pop();
return Ok((Tok::Dedent, TextRange::empty(self.offset())));
match self.indentations.current().try_compare(indentation) {
Ok(Ordering::Greater) => {
self.pending_indentation = Some(indentation);
let offset = self.offset();
self.indentations.dedent_one(indentation).map_err(|_| {
LexicalError::new(LexicalErrorType::IndentationError, offset)
})?;
return Ok((Tok::Dedent, TextRange::empty(offset)));
}
Ok(_) => {}
Err(_) => {
return Err(LexicalError::new(
LexicalErrorType::IndentationError,
self.offset(),
));
}
}
}

Expand Down Expand Up @@ -690,17 +702,20 @@ impl<'source> Lexer<'source> {
let token = match self.indentations.current().try_compare(indentation) {
// Dedent
Ok(Ordering::Greater) => {
self.indentations.pop();
self.pending_indentation = Some(indentation);

self.indentations.dedent_one(indentation).map_err(|_| {
LexicalError::new(LexicalErrorType::IndentationError, self.offset())
})?;

Some((Tok::Dedent, TextRange::empty(self.offset())))
}

Ok(Ordering::Equal) => None,

// Indent
Ok(Ordering::Less) => {
self.indentations.push(indentation);
self.indentations.indent(indentation);
Some((Tok::Indent, self.token_range()))
}
Err(_) => {
Expand Down Expand Up @@ -732,7 +747,7 @@ impl<'source> Lexer<'source> {
Ok((Tok::Newline, TextRange::empty(self.offset())))
}
// Next, flush the indentation stack to zero.
else if self.indentations.pop().is_some() {
else if self.indentations.dedent().is_some() {
Ok((Tok::Dedent, TextRange::empty(self.offset())))
} else {
Ok((Tok::EndOfFile, TextRange::empty(self.offset())))
Expand Down Expand Up @@ -1656,7 +1671,7 @@ def f(arg=%timeit a = b):
// This test case is to just make sure that the lexer doesn't go into
// infinite loop on invalid input.
#[test]
fn test_infite_loop() {
fn test_infinite_loop() {
let source = "[1";
let _ = lex(source, Mode::Module).collect::<Vec<_>>();
}
Expand All @@ -1678,4 +1693,16 @@ def f(arg=%timeit a = b):
result => panic!("Expected an error token but found {result:?}"),
}
}

#[test]
fn tet_too_low_dedent() {
let tokens: Vec<_> = lex(
r#"if True:
pass
pass"#,
Mode::Module,
)
.collect();
assert_debug_snapshot!(tokens);
}
}
24 changes: 22 additions & 2 deletions crates/ruff_python_parser/src/lexer/indentation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,33 @@ pub(super) struct Indentations {
}

impl Indentations {
pub(super) fn push(&mut self, indent: Indentation) {
pub(super) fn indent(&mut self, indent: Indentation) {
debug_assert_eq!(self.current().try_compare(indent), Ok(Ordering::Less));

self.stack.push(indent);
}

pub(super) fn pop(&mut self) -> Option<Indentation> {
/// Dedent one level to eventually reach `new_indentation`.
///
/// Returns `Err` if the `new_indentation` is greater than the new current indentation level.
pub(super) fn dedent_one(
&mut self,
new_indentation: Indentation,
) -> Result<Option<Indentation>, UnexpectedIndentation> {
let previous = self.dedent();

match new_indentation.try_compare(*self.current())? {
Ordering::Less | Ordering::Equal => Ok(previous),
// ```python
// if True:
// pass
// pass <- The indentation is greater than the expected indent of 0.
// ```
Ordering::Greater => Err(UnexpectedIndentation),
}
}

pub(super) fn dedent(&mut self) -> Option<Indentation> {
self.stack.pop()
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: tokens
---
[
Ok(
(
If,
0..2,
),
),
Ok(
(
True,
3..7,
),
),
Ok(
(
Colon,
7..8,
),
),
Ok(
(
Newline,
8..9,
),
),
Ok(
(
Indent,
9..13,
),
),
Ok(
(
Pass,
13..17,
),
),
Ok(
(
Newline,
17..18,
),
),
Err(
LexicalError {
error: IndentationError,
location: 20,
},
),
Ok(
(
Pass,
20..24,
),
),
Ok(
(
Newline,
24..24,
),
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: "lex_source(r#\"if True:\n pass\npass\"#)"
---
[
(
If,
0..2,
),
(
True,
3..7,
),
(
Colon,
7..8,
),
(
Newline,
8..9,
),
(
Indent,
9..13,
),
(
Pass,
13..17,
),
(
Newline,
17..18,
),
(
Dedent,
18..18,
),
(
Pass,
18..22,
),
(
Newline,
22..22,
),
]

0 comments on commit 67b69ed

Please sign in to comment.