From d3afcc4e7cec7206ffbba7ac630985128cbe57ae Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sat, 4 Mar 2023 12:18:31 +0530 Subject: [PATCH 1/8] Support files with type comment syntax errors --- src/black/parsing.py | 16 ++++++++++++---- .../type_comments/type_comment_syntax_error.py | 11 +++++++++++ tests/test_black.py | 8 ++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 tests/data/type_comments/type_comment_syntax_error.py diff --git a/src/black/parsing.py b/src/black/parsing.py index ba474c5e047..ee88e708703 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -148,18 +148,20 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( - src: str, version: Tuple[int, int] + src: str, version: Tuple[int, int], type_comments: bool ) -> Union[ast.AST, ast3.AST]: filename = "" # typed-ast is needed because of feature version limitations in the builtin ast 3.8> if sys.version_info >= (3, 8) and version >= (3,): - return ast.parse(src, filename, feature_version=version, type_comments=True) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) if _IS_PYPY: # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's # not much we can do as typed-ast won't work either. if sys.version_info >= (3, 8): - return ast3.parse(src, filename, type_comments=True) + return ast3.parse(src, filename, type_comments=type_comments) else: return ast3.parse(src, filename) else: @@ -175,11 +177,17 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: first_error = "" for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version) + return parse_single_version(src, version, type_comments=True) except SyntaxError as e: if not first_error: first_error = str(e) + # Try to parse without type comments + try: + return parse_single_version(src, version, type_comments=False) + except SyntaxError: + pass + raise SyntaxError(first_error) diff --git a/tests/data/type_comments/type_comment_syntax_error.py b/tests/data/type_comments/type_comment_syntax_error.py new file mode 100644 index 00000000000..2e5ca2ede8c --- /dev/null +++ b/tests/data/type_comments/type_comment_syntax_error.py @@ -0,0 +1,11 @@ +def foo( + # type: Foo + x): pass + +# output + +def foo( + # type: Foo + x, +): + pass diff --git a/tests/test_black.py b/tests/test_black.py index e5e17777715..d07a84d477c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1938,6 +1938,14 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: err.match("invalid character") err.match(r"\(, line 1\)") + def test_type_comment_syntax_error(self) -> None: + """ + Test that black is able to format python code with type comment syntax errors. + """ + source, expected = read_data("type_comments", "type_comment_syntax_error") + assert_format(source, expected) + black.assert_equivalent(source, expected) + class TestCaching: def test_get_cache_dir( From 4353904c976daa673770905828b568a2b40a0cd4 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sat, 4 Mar 2023 12:26:27 +0530 Subject: [PATCH 2/8] Add changelog entry --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a8b556cb7e8..bdc1459d6e2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,8 @@ +- Added support for formatting files with invalid type comments (#3594) + ### Performance From 69640562aa398f528856ce91889330a10dfa83bf Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sat, 4 Mar 2023 22:38:52 +0530 Subject: [PATCH 3/8] Fix typed_ast usage in <=3.7 --- src/black/parsing.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index ee88e708703..07c7a4f8148 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -165,9 +165,12 @@ def parse_single_version( else: return ast3.parse(src, filename) else: - # Typed-ast is guaranteed to be used here and automatically tracks type - # comments separately. - return ast3.parse(src, filename, feature_version=version[1]) + if type_comments: + # Typed-ast is guaranteed to be used here and automatically tracks type + # comments separately. + return ast3.parse(src, filename, feature_version=version[1]) + else: + return ast.parse(src, filename) def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: From 3549c658c37ab4ca964c979686da0015f4b3d2f6 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 5 Mar 2023 01:04:23 +0530 Subject: [PATCH 4/8] Move test to test_format --- tests/test_black.py | 8 -------- tests/test_format.py | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index d07a84d477c..e5e17777715 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1938,14 +1938,6 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: err.match("invalid character") err.match(r"\(, line 1\)") - def test_type_comment_syntax_error(self) -> None: - """ - Test that black is able to format python code with type comment syntax errors. - """ - source, expected = read_data("type_comments", "type_comment_syntax_error") - assert_format(source, expected) - black.assert_equivalent(source, expected) - class TestCaching: def test_get_cache_dir( diff --git a/tests/test_format.py b/tests/test_format.py index ab849aac9a3..f8ada9bd470 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -190,3 +190,9 @@ def test_power_op_newline() -> None: # requires line_length=0 source, expected = read_data("miscellaneous", "power_op_newline") assert_format(source, expected, mode=black.Mode(line_length=0)) + +def test_type_comment_syntax_error() -> None: + """Test that black is able to format python code with type comment syntax errors.""" + source, expected = read_data("type_comments", "type_comment_syntax_error") + assert_format(source, expected) + black.assert_equivalent(source, expected) From cd7d9d9338b9ad52e9bad627fb251f1f7d65e069 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 5 Mar 2023 01:05:01 +0530 Subject: [PATCH 5/8] Make type_comments a keyword-only argument --- src/black/parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index 07c7a4f8148..479ab069a7a 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -148,7 +148,7 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( - src: str, version: Tuple[int, int], type_comments: bool + src: str, version: Tuple[int, int], *, type_comments: bool ) -> Union[ast.AST, ast3.AST]: filename = "" # typed-ast is needed because of feature version limitations in the builtin ast 3.8> From 911d910fcb63d26410f3c86c87dc26872c950106 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 5 Mar 2023 21:01:56 +0530 Subject: [PATCH 6/8] Move new code to separate for loop --- src/black/parsing.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index 479ab069a7a..4ddb5d18b3a 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -185,11 +185,13 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: if not first_error: first_error = str(e) - # Try to parse without type comments - try: - return parse_single_version(src, version, type_comments=False) - except SyntaxError: - pass + + # Try to parse without type comments + for version in sorted(versions, reverse=True): + try: + return parse_single_version(src, version, type_comments=False) + except SyntaxError: + pass raise SyntaxError(first_error) From acf77e41950af68037bff7c781db0160bd873917 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 5 Mar 2023 21:04:41 +0530 Subject: [PATCH 7/8] run black on file --- src/black/parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index 4ddb5d18b3a..eaa3c367e54 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -185,7 +185,6 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: if not first_error: first_error = str(e) - # Try to parse without type comments for version in sorted(versions, reverse=True): try: From 7f66d48dd79906bdb0b206bd3adef462e49b29b9 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 5 Mar 2023 21:05:00 +0530 Subject: [PATCH 8/8] Run black on test file --- tests/test_format.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_format.py b/tests/test_format.py index f8ada9bd470..81770bbf3cb 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -191,6 +191,7 @@ def test_power_op_newline() -> None: source, expected = read_data("miscellaneous", "power_op_newline") assert_format(source, expected, mode=black.Mode(line_length=0)) + def test_type_comment_syntax_error() -> None: """Test that black is able to format python code with type comment syntax errors.""" source, expected = read_data("type_comments", "type_comment_syntax_error")