From a66606edf1aa5e5b9103536e280cc4a8447bc0bf Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 16 Oct 2023 07:04:30 -0700 Subject: [PATCH 1/3] Fix parser bug where "type" was misinterpreted as a keyword inside a match Fixes #3790 Slightly hacky, but I think this is correct and it should also improve performance somewhat. --- CHANGES.md | 3 +++ src/blib2to3/pgen2/parse.py | 19 ++++++++++++++++++- tests/data/cases/pattern_matching_complex.py | 4 ++++ tests/data/cases/type_aliases.py | 9 +++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a608551815f..8a95c3ef81c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,6 +35,9 @@ +- Fix bug where attributes named `type` were not acccepted inside `match` + statements (#3950) + ### Performance diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index 299cc24a15f..ad51a3dad08 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -211,6 +211,7 @@ def __init__(self, grammar: Grammar, convert: Optional[Convert] = None) -> None: # See note in docstring above. TL;DR this is ignored. self.convert = convert or lam_sub self.is_backtracking = False + self.last_token: Optional[int] = None def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: """Prepare for parsing. @@ -236,6 +237,7 @@ def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: self.rootnode: Optional[NL] = None self.used_names: Set[str] = set() self.proxy = proxy + self.last_token = None def addtoken(self, type: int, value: str, context: Context) -> bool: """Add a token; return True iff this is the end of the program.""" @@ -317,6 +319,7 @@ def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> boo dfa, state, node = self.stack[-1] states, first = dfa # Done with this token + self.last_token = type return False else: @@ -343,9 +346,23 @@ def classify(self, type: int, value: str, context: Context) -> List[int]: return [self.grammar.keywords[value]] elif value in self.grammar.soft_keywords: assert type in self.grammar.tokens + # Current soft keywords (match, case, type) can only appear at the + # beginning of a statement. So as a shortcut, don't try to treat them + # like keywords in any other context. + # ('_' is also a soft keyword in the real grammar, but for our grammar + # it's just an expression, so we don't need to treat it specially.) + if self.last_token not in ( + None, + token.INDENT, + token.DEDENT, + token.NEWLINE, + token.SEMI, + token.COLON, + ): + return [self.grammar.tokens[type]] return [ - self.grammar.soft_keywords[value], self.grammar.tokens[type], + self.grammar.soft_keywords[value], ] ilabel = self.grammar.tokens.get(type) diff --git a/tests/data/cases/pattern_matching_complex.py b/tests/data/cases/pattern_matching_complex.py index b4355c7333a..10b4d26e289 100644 --- a/tests/data/cases/pattern_matching_complex.py +++ b/tests/data/cases/pattern_matching_complex.py @@ -143,3 +143,7 @@ y = 1 case []: y = 2 +# issue 3790 +match (X.type, Y): + case _: + pass diff --git a/tests/data/cases/type_aliases.py b/tests/data/cases/type_aliases.py index a3c1931c9fc..52f5c4e265e 100644 --- a/tests/data/cases/type_aliases.py +++ b/tests/data/cases/type_aliases.py @@ -1,6 +1,8 @@ # flags: --minimum-version=3.12 type A=int type Gen[T]=list[T] +type One = int; type Another = str +class X: type InClass = int type = aliased print(type(42)) @@ -9,6 +11,13 @@ type A = int type Gen[T] = list[T] +type One = int +type Another = str + + +class X: + type InClass = int + type = aliased print(type(42)) From 4e8bcf8bed39b6781f714036dea3972e90119a0d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:10:07 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8a95c3ef81c..bea1a1987b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,8 +35,8 @@ -- Fix bug where attributes named `type` were not acccepted inside `match` - statements (#3950) +- Fix bug where attributes named `type` were not acccepted inside `match` statements + (#3950) ### Performance From d14db1b7788b5a58a6ac614363d5e406101e388b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 16 Oct 2023 11:05:22 -0700 Subject: [PATCH 3/3] fix test --- tests/data/cases/type_aliases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/cases/type_aliases.py b/tests/data/cases/type_aliases.py index 358b1e45c9f..7c2009e8202 100644 --- a/tests/data/cases/type_aliases.py +++ b/tests/data/cases/type_aliases.py @@ -25,5 +25,6 @@ class X: type InClass = int class X: type InClass = int + type = aliased print(type(42))