From d8f05f572feb9798d9692096615f02588b5700f1 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Thu, 12 Oct 2023 15:09:21 -0700 Subject: [PATCH] Rebase _Pyink_ to _Black_ v23.9.1. Notably the the preview style for "dummy implementations" is disabled in Pyink as it doesn't play well with our linter (yet). The change in _Black_ is from https://github.com/psf/black/pull/3796. PiperOrigin-RevId: 573023762 --- CHANGES.md | 4 +- patches/pyink.patch | 284 ++++++----- pyproject.toml | 17 +- src/pyink/__init__.py | 147 +++--- src/pyink/_width_table.py | 8 +- src/pyink/brackets.py | 8 +- src/pyink/cache.py | 166 ++++--- src/pyink/comments.py | 8 +- src/pyink/concurrency.py | 12 +- src/pyink/const.py | 2 +- src/pyink/files.py | 55 ++- src/pyink/handle_ipynb_magics.py | 31 +- src/pyink/linegen.py | 39 +- src/pyink/lines.py | 66 ++- src/pyink/mode.py | 36 +- src/pyink/nodes.py | 48 +- src/pyink/parsing.py | 118 ++--- src/pyink/strings.py | 10 +- src/pyink/trans.py | 103 ++-- test_requirements.txt | 4 +- tests/data/miscellaneous/force_py36.py | 4 +- .../data/miscellaneous/nested_class_stub.pyi | 16 - tests/data/miscellaneous/nested_stub.pyi | 43 ++ tests/data/preview/comments7.py | 3 +- tests/data/preview/dummy_implementations.py | 99 ++++ .../preview/no_blank_line_before_docstring.py | 58 +++ tests/data/preview/pep_572.py | 6 + tests/data/preview_py_310/pep_572.py | 12 + tests/data/py_312/type_aliases.py | 13 + tests/data/py_312/type_params.py | 57 +++ tests/data/simple_cases/comments2.py | 5 + tests/data/simple_cases/fstring.py | 4 + tests/data/simple_cases/ignore_pyi.py | 41 ++ ...ine_consecutive_open_parentheses_ignore.py | 41 ++ tests/data/simple_cases/pep_604.py | 25 + tests/test_black.py | 467 +++++++++++------- tests/test_format.py | 23 +- tests/test_ipynb.py | 27 +- tests/test_no_ipynb.py | 3 +- tests/test_trans.py | 2 +- tox.ini | 12 +- 41 files changed, 1350 insertions(+), 777 deletions(-) delete mode 100644 tests/data/miscellaneous/nested_class_stub.pyi create mode 100644 tests/data/miscellaneous/nested_stub.pyi create mode 100644 tests/data/preview/dummy_implementations.py create mode 100644 tests/data/preview/no_blank_line_before_docstring.py create mode 100644 tests/data/preview/pep_572.py create mode 100644 tests/data/preview_py_310/pep_572.py create mode 100644 tests/data/py_312/type_aliases.py create mode 100644 tests/data/py_312/type_params.py create mode 100644 tests/data/simple_cases/ignore_pyi.py create mode 100644 tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py create mode 100644 tests/data/simple_cases/pep_604.py diff --git a/CHANGES.md b/CHANGES.md index 33a2728d84f..15c62e2e7b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,9 @@ All notable changes to Pyink are recorded here. ## Unreleased -* Nothing notable unreleased. +* _Black_ v23.9.9's new concise formatting for dummy implementations from + [psf/black#3796](https://github.com/psf/black/pull/3796) is temporarily + disabled for _Pyink_. ## 23.5.0 diff --git a/patches/pyink.patch b/patches/pyink.patch index e987dd74cf4..8a58fcb0db0 100644 --- a/patches/pyink.patch +++ b/patches/pyink.patch @@ -10,15 +10,16 @@ Dict, Generator, Iterator, -@@ -68,6 +69,7 @@ from pyink.mode import ( - VERSION_TO_FEATURES, - Feature, - Mode, -+ QuoteStyle, - TargetVersion, - supports_feature, - ) -@@ -81,6 +83,8 @@ from pyink.nodes import ( +@@ -65,7 +66,7 @@ from pyink.linegen import LN, LineGenera + from pyink.lines import EmptyLineTracker, LinesBlock + from pyink.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature + from pyink.mode import Mode as Mode # re-exported +-from pyink.mode import TargetVersion, supports_feature ++from pyink.mode import QuoteStyle, TargetVersion, supports_feature + from pyink.nodes import ( + STARS, + is_number_token, +@@ -76,6 +77,8 @@ from pyink.nodes import ( from pyink.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from pyink.parsing import InvalidInput # noqa F401 from pyink.parsing import lib2to3_parse, parse_ast, stringify_ast @@ -27,7 +28,7 @@ from pyink.report import Changed, NothingChanged, Report from pyink.trans import iter_fexpr_spans from blib2to3.pgen2 import token -@@ -284,6 +288,41 @@ def validate_regex( +@@ -291,6 +294,41 @@ def validate_regex( ), ) @click.option( @@ -69,7 +70,7 @@ "--check", is_flag=True, help=( -@@ -442,6 +481,10 @@ def main( # noqa: C901 +@@ -453,6 +491,10 @@ def main( # noqa: C901 skip_magic_trailing_comma: bool, experimental_string_processing: bool, preview: bool, @@ -80,7 +81,7 @@ quiet: bool, verbose: bool, required_version: Optional[str], -@@ -540,6 +583,7 @@ def main( # noqa: C901 +@@ -531,6 +573,7 @@ def main( # noqa: C901 else: # We'll autodetect later. versions = set() @@ -88,7 +89,7 @@ mode = Mode( target_versions=versions, line_length=line_length, -@@ -551,8 +595,36 @@ def main( # noqa: C901 +@@ -542,8 +585,36 @@ def main( # noqa: C901 experimental_string_processing=experimental_string_processing, preview=preview, python_cell_magics=set(python_cell_magics), @@ -125,7 +126,7 @@ if code is not None: # Run in quiet mode by default with -c; the extra output isn't useful. # You can still pass -v to get verbose output. -@@ -596,6 +668,7 @@ def main( # noqa: C901 +@@ -588,6 +659,7 @@ def main( # noqa: C901 write_back=write_back, mode=mode, report=report, @@ -133,7 +134,7 @@ ) else: from pyink.concurrency import reformat_many -@@ -740,7 +813,13 @@ def reformat_code( +@@ -745,7 +817,13 @@ def reformat_code( # not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26 @mypyc_attr(patchable=True) def reformat_one( @@ -148,7 +149,7 @@ ) -> None: """Reformat a single file under `src` without spawning child processes. -@@ -765,7 +844,9 @@ def reformat_one( +@@ -770,7 +848,9 @@ def reformat_one( mode = replace(mode, is_pyi=True) elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) @@ -158,9 +159,9 @@ + ): changed = Changed.YES else: - cache: Cache = {} -@@ -776,7 +857,7 @@ def reformat_one( - if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src): + cache = Cache.read(mode) +@@ -778,7 +858,7 @@ def reformat_one( + if not cache.is_changed(src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( - src, fast=fast, write_back=write_back, mode=mode @@ -168,7 +169,7 @@ ): changed = Changed.YES if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( -@@ -796,6 +877,8 @@ def format_file_in_place( +@@ -798,6 +878,8 @@ def format_file_in_place( mode: Mode, write_back: WriteBack = WriteBack.NO, lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy @@ -177,7 +178,7 @@ ) -> bool: """Format file under `src` path. Return True if changed. -@@ -815,7 +898,9 @@ def format_file_in_place( +@@ -817,7 +899,9 @@ def format_file_in_place( header = buf.readline() src_contents, encoding, newline = decode_bytes(buf.read()) try: @@ -188,7 +189,7 @@ except NothingChanged: return False except JSONDecodeError: -@@ -860,6 +945,7 @@ def format_stdin_to_stdout( +@@ -862,6 +946,7 @@ def format_stdin_to_stdout( content: Optional[str] = None, write_back: WriteBack = WriteBack.NO, mode: Mode, @@ -196,7 +197,7 @@ ) -> bool: """Format file on stdin. Return True if changed. -@@ -878,7 +964,7 @@ def format_stdin_to_stdout( +@@ -880,7 +965,7 @@ def format_stdin_to_stdout( dst = src try: @@ -205,7 +206,7 @@ return True except NothingChanged: -@@ -906,7 +992,11 @@ def format_stdin_to_stdout( +@@ -908,7 +993,11 @@ def format_stdin_to_stdout( def check_stability_and_equivalence( @@ -218,7 +219,7 @@ ) -> None: """Perform stability and equivalence checks. -@@ -915,10 +1005,16 @@ def check_stability_and_equivalence( +@@ -917,10 +1006,16 @@ def check_stability_and_equivalence( content differently. """ assert_equivalent(src_contents, dst_contents) @@ -237,7 +238,7 @@ """Reformat contents of a file and return new contents. If `fast` is False, additionally confirm that the reformatted code is -@@ -928,13 +1024,15 @@ def format_file_contents(src_contents: s +@@ -930,13 +1025,15 @@ def format_file_contents(src_contents: s if mode.is_ipynb: dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) else: @@ -255,7 +256,7 @@ return dst_contents -@@ -1045,7 +1143,12 @@ def format_ipynb_string(src_contents: st +@@ -1047,7 +1144,12 @@ def format_ipynb_string(src_contents: st raise NothingChanged @@ -269,7 +270,7 @@ """Reformat a string and return new contents. `mode` determines formatting options, such as how many characters per line are -@@ -1074,17 +1177,28 @@ def format_str(src_contents: str, *, mod +@@ -1076,17 +1178,28 @@ def format_str(src_contents: str, *, mod ) -> None: hey @@ -301,7 +302,7 @@ src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) dst_blocks: List[LinesBlock] = [] if mode.target_versions: -@@ -1093,6 +1207,8 @@ def _format_str_once(src_contents: str, +@@ -1095,6 +1208,8 @@ def _format_str_once(src_contents: str, future_imports = get_future_imports(src_node) versions = detect_target_versions(src_node, future_imports=future_imports) @@ -310,10 +311,10 @@ context_manager_features = { feature for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} -@@ -1101,7 +1217,12 @@ def _format_str_once(src_contents: str, +@@ -1103,7 +1218,12 @@ def _format_str_once(src_contents: str, + if supports_feature(versions, feature) + } normalize_fmt_off(src_node) - # This should be called after normalize_fmt_off but before convert_unchanged_lines. - import_sorting.sort_imports_in_place(src_node, reformat=True, safe_mode=True, lines=lines) - lines = LineGenerator(mode=mode, features=context_manager_features) + + if lines: @@ -324,7 +325,7 @@ elt = EmptyLineTracker(mode=mode) split_line_features = { feature -@@ -1109,7 +1230,7 @@ def _format_str_once(src_contents: str, +@@ -1111,7 +1231,7 @@ def _format_str_once(src_contents: str, if supports_feature(versions, feature) } block: Optional[LinesBlock] = None @@ -333,7 +334,7 @@ block = elt.maybe_empty_lines(current_line) dst_blocks.append(block) for line in transform_line( -@@ -1374,12 +1495,20 @@ def assert_equivalent(src: str, dst: str +@@ -1379,12 +1499,20 @@ def assert_equivalent(src: str, dst: str ) from None @@ -358,16 +359,16 @@ str(mode), --- a/_width_table.py +++ b/_width_table.py -@@ -9,7 +9,7 @@ if sys.version_info < (3, 8): - else: - from typing import Final +@@ -3,7 +3,7 @@ + # Unicode 15.0.0 + from typing import Final, List, Tuple -WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ +WIDTH_TABLE: Final[Tuple[Tuple[int, int, int], ...]] = ( (0, 0, 0), (1, 31, -1), (127, 159, -1), -@@ -481,4 +481,4 @@ WIDTH_TABLE: Final[List[Tuple[int, int, +@@ -475,4 +475,4 @@ WIDTH_TABLE: Final[List[Tuple[int, int, (131072, 196605, 2), (196608, 262141, 2), (917760, 917999, 0), @@ -395,7 +396,7 @@ Line, RHSResult, append_leaves, -@@ -82,6 +88,15 @@ LeafID = int +@@ -83,6 +89,15 @@ LeafID = int LN = Union[Leaf, Node] @@ -411,7 +412,7 @@ class CannotSplit(CannotTransform): """A readable split that fits the allotted line length is impossible.""" -@@ -101,7 +116,9 @@ class LineGenerator(Visitor[Line]): +@@ -102,7 +117,9 @@ class LineGenerator(Visitor[Line]): self.current_line: Line self.__post_init__() @@ -422,7 +423,7 @@ """Generate a line. If the line is empty, only emit if it makes sense. -@@ -110,7 +127,10 @@ class LineGenerator(Visitor[Line]): +@@ -111,7 +128,10 @@ class LineGenerator(Visitor[Line]): If any lines were generated, set up a new current_line. """ if not self.current_line: @@ -434,7 +435,7 @@ return # Line is empty, don't emit. Creating a new one unnecessary. if ( -@@ -125,7 +145,13 @@ class LineGenerator(Visitor[Line]): +@@ -126,7 +146,13 @@ class LineGenerator(Visitor[Line]): return complete_line = self.current_line @@ -449,7 +450,7 @@ yield complete_line def visit_default(self, node: LN) -> Iterator[Line]: -@@ -151,7 +177,9 @@ class LineGenerator(Visitor[Line]): +@@ -152,7 +178,9 @@ class LineGenerator(Visitor[Line]): normalize_prefix(node, inside_brackets=any_open_brackets) if self.mode.string_normalization and node.type == token.STRING: node.value = normalize_string_prefix(node.value) @@ -460,7 +461,7 @@ if node.type == token.NUMBER: normalize_numeric_literal(node) if node.type not in WHITESPACE: -@@ -161,7 +189,10 @@ class LineGenerator(Visitor[Line]): +@@ -162,7 +190,10 @@ class LineGenerator(Visitor[Line]): def visit_test(self, node: Node) -> Iterator[Line]: """Visit an `x if y else z` test""" @@ -472,7 +473,7 @@ already_parenthesized = ( node.prev_sibling and node.prev_sibling.type == token.LPAR ) -@@ -177,7 +208,7 @@ class LineGenerator(Visitor[Line]): +@@ -178,7 +209,7 @@ class LineGenerator(Visitor[Line]): def visit_INDENT(self, node: Leaf) -> Iterator[Line]: """Increase indentation level, maybe yield a line.""" # In blib2to3 INDENT never holds comments. @@ -481,7 +482,7 @@ yield from self.visit_default(node) def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: -@@ -192,7 +223,7 @@ class LineGenerator(Visitor[Line]): +@@ -193,7 +224,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) # Finally, emit the dedent. @@ -490,8 +491,8 @@ def visit_stmt( self, node: Node, keywords: Set[str], parens: Set[str] -@@ -288,9 +319,9 @@ class LineGenerator(Visitor[Line]): - if self.mode.is_pyi and is_stub_body(node): +@@ -305,9 +336,9 @@ class LineGenerator(Visitor[Line]): + ) and is_stub_body(node): yield from self.visit_default(node) else: - yield from self.line(+1) @@ -502,7 +503,7 @@ else: if ( -@@ -383,10 +414,13 @@ class LineGenerator(Visitor[Line]): +@@ -400,10 +431,13 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: @@ -518,7 +519,7 @@ # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: -@@ -397,7 +431,9 @@ class LineGenerator(Visitor[Line]): +@@ -414,7 +448,9 @@ class LineGenerator(Visitor[Line]): # formatting as visit_default() is called *after*. To avoid a # situation where this function formats a docstring differently on # the second pass, normalize it early. @@ -529,7 +530,7 @@ else: docstring = leaf.value prefix = get_string_prefix(docstring) -@@ -411,8 +447,9 @@ class LineGenerator(Visitor[Line]): +@@ -428,8 +464,9 @@ class LineGenerator(Visitor[Line]): quote_len = 1 if docstring[1] != quote_char else 3 docstring = docstring[quote_len:-quote_len] docstring_started_empty = not docstring @@ -540,7 +541,7 @@ if is_multiline_string(leaf): docstring = fix_docstring(docstring, indent) else: -@@ -453,7 +490,13 @@ class LineGenerator(Visitor[Line]): +@@ -470,7 +507,13 @@ class LineGenerator(Visitor[Line]): # If docstring is one line, we don't put the closing quotes on a # separate line because it looks ugly (#3320). lines = docstring.splitlines() @@ -555,7 +556,7 @@ # If adding closing quotes would cause the last line to exceed # the maximum line length then put a line break before the -@@ -465,6 +508,15 @@ class LineGenerator(Visitor[Line]): +@@ -482,6 +525,15 @@ class LineGenerator(Visitor[Line]): and not has_trailing_backslash ): leaf.value = prefix + quote + docstring + "\n" + indent + quote @@ -571,7 +572,7 @@ else: leaf.value = prefix + quote + docstring + quote else: -@@ -492,7 +544,8 @@ class LineGenerator(Visitor[Line]): +@@ -509,7 +561,8 @@ class LineGenerator(Visitor[Line]): self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) @@ -581,7 +582,7 @@ self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) self.visit_async_funcdef = self.visit_async_stmt self.visit_decorated = self.visit_decorators -@@ -519,10 +572,19 @@ def transform_line( +@@ -536,10 +589,19 @@ def transform_line( ll = mode.line_length sn = mode.string_normalization @@ -605,7 +606,7 @@ transformers: List[Transformer] if ( -@@ -697,8 +759,7 @@ def _first_right_hand_split( +@@ -714,8 +776,7 @@ def _first_right_hand_split( omit: Collection[LeafID] = (), ) -> RHSResult: """Split the line into head, body, tail starting with the last bracket pair. @@ -615,7 +616,7 @@ _maybe_split_omitting_optional_parens to get an opinion whether to prefer splitting on the right side of an assignment statement. """ -@@ -727,12 +788,53 @@ def _first_right_hand_split( +@@ -744,12 +805,53 @@ def _first_right_hand_split( tail_leaves.reverse() body_leaves.reverse() head_leaves.reverse() @@ -672,7 +673,7 @@ tail = bracket_split_build_line( tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail ) -@@ -891,7 +993,7 @@ def bracket_split_build_line( +@@ -908,7 +1010,7 @@ def bracket_split_build_line( result = Line(mode=original.mode, depth=original.depth) if component is _BracketSplitComponent.body: result.inside_brackets = True @@ -681,9 +682,9 @@ if leaves: # Since body is a new indent level, remove spurious leading whitespace. normalize_prefix(leaves[0], inside_brackets=True) -@@ -915,6 +1017,13 @@ def bracket_split_build_line( - ) - if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf) +@@ -939,6 +1041,13 @@ def bracket_split_build_line( + and leaves[0].parent.next_sibling + and leaves[0].parent.next_sibling.type == token.VBAR ) + # Except the false negatives above for PEP 604 unions where we + # can't add the comma. @@ -695,7 +696,7 @@ ) if original.is_import or no_commas: -@@ -1449,7 +1558,7 @@ def generate_trailers_to_omit(line: Line +@@ -1478,7 +1587,7 @@ def generate_trailers_to_omit(line: Line if not line.magic_trailing_comma: yield omit @@ -714,7 +715,7 @@ import sys from dataclasses import dataclass, field from typing import ( -@@ -43,13 +45,28 @@ Index = int +@@ -44,13 +46,28 @@ Index = int LeafID = int LN = Union[Leaf, Node] @@ -738,13 +739,13 @@ class Line: """Holds leaves and comments. Can be printed with `str(line)`.""" - mode: Mode + mode: Mode = field(repr=False) - depth: int = 0 + depth: Tuple[Indentation, ...] = field(default_factory=tuple) leaves: List[Leaf] = field(default_factory=list) # keys ordered like `leaves` comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict) -@@ -58,6 +75,9 @@ class Line: +@@ -59,6 +76,9 @@ class Line: should_split_rhs: bool = False magic_trailing_comma: Optional[Leaf] = None @@ -754,7 +755,7 @@ def append( self, leaf: Leaf, preformatted: bool = False, track_bracket: bool = False ) -> None: -@@ -98,7 +118,7 @@ class Line: +@@ -101,7 +121,7 @@ class Line: Raises ValueError when any `leaf` is appended after a standalone comment or when a standalone comment is not the first leaf on the line. """ @@ -763,7 +764,7 @@ if self.is_comment: raise ValueError("cannot append to standalone comments") -@@ -293,6 +313,20 @@ class Line: +@@ -303,6 +323,20 @@ class Line: return False @@ -784,7 +785,7 @@ def contains_multiline_strings(self) -> bool: return any(is_multiline_string(leaf) for leaf in self.leaves) -@@ -460,7 +494,7 @@ class Line: +@@ -470,7 +504,7 @@ class Line: if not self: return "\n" @@ -793,7 +794,7 @@ leaves = iter(self.leaves) first = next(leaves) res = f"{first.prefix}{indent}{first.value}" -@@ -567,7 +601,7 @@ class EmptyLineTracker: +@@ -577,7 +611,7 @@ class EmptyLineTracker: def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: max_allowed = 1 @@ -802,16 +803,16 @@ max_allowed = 1 if self.mode.is_pyi else 2 if current_line.leaves: # Consume the first leaf's extra newlines. -@@ -578,7 +612,7 @@ class EmptyLineTracker: - else: - before = 0 +@@ -592,7 +626,7 @@ class EmptyLineTracker: depth = current_line.depth + + previous_def = None - while self.previous_defs and self.previous_defs[-1].depth >= depth: + while self.previous_defs and len(self.previous_defs[-1].depth) >= len(depth): - if self.mode.is_pyi: - assert self.previous_line is not None - if depth and not current_line.is_def and self.previous_line.is_def: -@@ -622,10 +656,25 @@ class EmptyLineTracker: + previous_def = self.previous_defs.pop() + + if previous_def is not None: +@@ -641,10 +675,25 @@ class EmptyLineTracker: if ( self.previous_line @@ -839,8 +840,8 @@ ): return (before or 1), 0 -@@ -636,6 +685,14 @@ class EmptyLineTracker: - ): +@@ -657,6 +706,14 @@ class EmptyLineTracker: + return 0, 1 return before, 1 + if ( @@ -854,7 +855,7 @@ if self.previous_line and self.previous_line.opens_block: return 0, 0 return before, 0 -@@ -656,15 +713,16 @@ class EmptyLineTracker: +@@ -677,15 +734,16 @@ class EmptyLineTracker: return 0, 0 @@ -874,7 +875,7 @@ and before == 0 ): slc = self.semantic_leading_comment -@@ -681,9 +739,9 @@ class EmptyLineTracker: +@@ -702,9 +760,9 @@ class EmptyLineTracker: if self.mode.is_pyi: if current_line.is_class or self.previous_line.is_class: @@ -886,7 +887,7 @@ newlines = 1 elif current_line.is_stub_class and self.previous_line.is_stub_class: # No blank line between classes with an empty body -@@ -701,7 +759,7 @@ class EmptyLineTracker: +@@ -733,7 +791,7 @@ class EmptyLineTracker: # Blank line between a block of functions (maybe with preceding # decorators) and a block of non-functions newlines = 1 @@ -895,7 +896,7 @@ newlines = 1 else: newlines = 0 -@@ -761,10 +819,14 @@ def is_line_short_enough( # noqa: C901 +@@ -801,10 +859,14 @@ def is_line_short_enough( # noqa: C901 line_str = line_to_string(line) width = str_width if mode.preview else len @@ -911,7 +912,7 @@ and "\n" not in line_str # multiline strings and not line.contains_standalone_comments() ) -@@ -773,7 +835,7 @@ def is_line_short_enough( # noqa: C901 +@@ -813,7 +875,7 @@ def is_line_short_enough( # noqa: C901 return False if "\n" not in line_str: # No multiline strings (MLS) present @@ -920,7 +921,7 @@ first, *_, last = line_str.split("\n") if width(first) > mode.line_length or width(last) > mode.line_length: -@@ -963,7 +1025,7 @@ def can_omit_invisible_parens( +@@ -1003,7 +1065,7 @@ def can_omit_invisible_parens( def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" remainder = False @@ -929,7 +930,7 @@ _index = -1 for _index, leaf, leaf_length in line.enumerate_with_length(): if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: -@@ -987,7 +1049,7 @@ def _can_omit_opening_paren(line: Line, +@@ -1027,7 +1089,7 @@ def _can_omit_opening_paren(line: Line, def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" @@ -940,19 +941,16 @@ length += leaf_length --- a/mode.py +++ b/mode.py -@@ -13,9 +13,9 @@ from typing import Dict, Set +@@ -8,7 +8,7 @@ from dataclasses import dataclass, field + from enum import Enum, auto + from hashlib import sha256 + from operator import attrgetter +-from typing import Dict, Final, Set ++from typing import Dict, Final, Literal, Set from warnings import warn - if sys.version_info < (3, 8): -- from typing_extensions import Final -+ from typing_extensions import Final, Literal - else: -- from typing import Final -+ from typing import Final, Literal - from pyink.const import DEFAULT_LINE_LENGTH - -@@ -172,11 +172,33 @@ class Deprecated(UserWarning): +@@ -191,11 +191,33 @@ class Deprecated(UserWarning): """Visible deprecation warning.""" @@ -986,7 +984,7 @@ is_pyi: bool = False is_ipynb: bool = False skip_source_first_line: bool = False -@@ -184,6 +206,8 @@ class Mode: +@@ -203,6 +225,8 @@ class Mode: experimental_string_processing: bool = False python_cell_magics: Set[str] = field(default_factory=set) preview: bool = False @@ -995,7 +993,17 @@ def __post_init__(self) -> None: if self.experimental_string_processing: -@@ -216,12 +240,25 @@ class Mode: +@@ -221,6 +245,9 @@ class Mode: + """ + if feature is Preview.string_processing: + return self.preview or self.experimental_string_processing ++ # dummy_implementations is temporarily disabled in Pyink. ++ if feature is Preview.dummy_implementations and self.is_pyink: ++ return False + return self.preview + + def get_cache_key(self) -> str: +@@ -235,12 +262,25 @@ class Mode: version_str, str(self.line_length), str(int(self.string_normalization)), @@ -1023,7 +1031,7 @@ + return Quote.DOUBLE --- a/nodes.py +++ b/nodes.py -@@ -521,7 +521,7 @@ def is_arith_like(node: LN) -> bool: +@@ -523,7 +523,7 @@ def is_arith_like(node: LN) -> bool: } @@ -1032,7 +1040,7 @@ if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): -@@ -533,6 +533,16 @@ def is_docstring(leaf: Leaf) -> bool: +@@ -535,6 +535,16 @@ def is_docstring(leaf: Leaf) -> bool: # grammar. We're safe to return True without further checks. return True @@ -1051,7 +1059,7 @@ --- a/pyproject.toml +++ b/pyproject.toml -@@ -1,51 +1,23 @@ +@@ -1,50 +1,23 @@ -# Example configuration for Black. - -# NOTE: you have to use single-quoted strings in TOML for regular expressions. @@ -1069,8 +1077,7 @@ -extend-exclude = ''' -/( - # The following are specific to Black, you probably don't want those. -- | blib2to3 -- | tests/data +- tests/data - | profiling -)/ -''' @@ -1094,7 +1101,7 @@ +name = "pyink" +description = "Pyink is a python formatter, forked from Black with slightly different behavior." license = { text = "MIT" } - requires-python = ">=3.7" + requires-python = ">=3.8" -authors = [ - { name = "Łukasz Langa", email = "lukasz@langa.pl" }, -] @@ -1112,11 +1119,11 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", -@@ -71,47 +43,34 @@ dependencies = [ +@@ -69,47 +42,34 @@ dependencies = [ + "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", - "typed-ast>=1.4.2; python_version < '3.8' and implementation_name == 'cpython'", - "typing_extensions>=3.10.0.0; python_version < '3.10'", -+ "black>=23.3.0", + "typing_extensions>=4.0.1; python_version < '3.11'", ++ "black>=23.9.1", ] -dynamic = ["readme", "version"] +dynamic = ["version"] @@ -1166,7 +1173,7 @@ [tool.hatch.build.targets.wheel] only-include = ["src"] sources = ["src"] -@@ -120,7 +79,6 @@ sources = ["src"] +@@ -118,7 +78,6 @@ sources = ["src"] # Option below requires `tests/optional.py` addopts = "--strict-config --strict-markers" optional-tests = [ @@ -1174,17 +1181,30 @@ "no_jupyter: run when `jupyter` extra NOT installed", ] markers = [ +@@ -142,12 +101,3 @@ filterwarnings = [ + # https://github.com/aio-libs/aiohttp/pull/7302 + "ignore:datetime.*utcfromtimestamp\\(\\) is deprecated and scheduled for removal:DeprecationWarning", + ] +-[tool.coverage.report] +-omit = [ +- "src/blib2to3/*", +- "tests/data/*", +- "*/site-packages/*", +- ".tox/*" +-] +-[tool.coverage.run] +-relative_files = true --- a/strings.py +++ b/strings.py -@@ -15,6 +15,7 @@ else: - from typing import Final +@@ -8,6 +8,7 @@ from functools import lru_cache + from typing import Final, List, Match, Pattern from pyink._width_table import WIDTH_TABLE +from pyink.mode import Quote + from blib2to3.pytree import Leaf STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. - STRING_PREFIX_RE: Final = re.compile( -@@ -176,8 +177,10 @@ def _cached_compile(pattern: str) -> Pat +@@ -170,8 +171,10 @@ def _cached_compile(pattern: str) -> Pat return re.compile(pattern) @@ -1197,7 +1217,7 @@ Adds or removes backslashes as appropriate. Doesn't parse and fix strings nested in f-strings. -@@ -244,8 +247,8 @@ def normalize_string_quotes(s: str) -> s +@@ -238,8 +241,8 @@ def normalize_string_quotes(s: str) -> s if new_escape_count > orig_escape_count: return s # Do not introduce more escaping @@ -1218,7 +1238,7 @@ +pyink = false --- a/tests/test_black.py +++ b/tests/test_black.py -@@ -1246,7 +1246,7 @@ class BlackTestCase(BlackBaseTestCase): +@@ -1243,7 +1243,7 @@ class BlackTestCase(BlackBaseTestCase): report=report, ) fsts.assert_called_once_with( @@ -1227,7 +1247,7 @@ ) # __PYINK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, pyink.Changed.YES) -@@ -1272,6 +1272,7 @@ class BlackTestCase(BlackBaseTestCase): +@@ -1269,6 +1269,7 @@ class BlackTestCase(BlackBaseTestCase): fast=True, write_back=pyink.WriteBack.YES, mode=replace(DEFAULT_MODE, is_pyi=True), @@ -1235,7 +1255,7 @@ ) # __PYINK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, pyink.Changed.YES) -@@ -1297,6 +1298,7 @@ class BlackTestCase(BlackBaseTestCase): +@@ -1294,6 +1295,7 @@ class BlackTestCase(BlackBaseTestCase): fast=True, write_back=pyink.WriteBack.YES, mode=replace(DEFAULT_MODE, is_ipynb=True), @@ -1243,7 +1263,7 @@ ) # __PYINK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, pyink.Changed.YES) -@@ -1383,6 +1385,28 @@ class BlackTestCase(BlackBaseTestCase): +@@ -1380,6 +1382,28 @@ class BlackTestCase(BlackBaseTestCase): pass # StringIO does not support detach assert output.getvalue() == "" @@ -1272,7 +1292,7 @@ def test_invalid_cli_regex(self) -> None: for option in ["--include", "--exclude", "--extend-exclude", "--force-exclude"]: self.invokeBlack(["-", option, "**()(!!*)"], exit_code=2) -@@ -2417,6 +2441,119 @@ class TestFileCollection: +@@ -2481,6 +2505,119 @@ class TestFileCollection: stdin_filename=stdin_filename, ) @@ -1390,11 +1410,11 @@ + assert diff in self.decode_and_normalized(result.stdout_bytes) + - try: - with open(pyink.__file__, "r", encoding="utf-8") as _bf: + class TestDeFactoAPI: + """Test that certain symbols that are commonly used externally keep working. --- a/tests/test_format.py +++ b/tests/test_format.py -@@ -43,6 +43,15 @@ def test_preview_format(filename: str) - +@@ -44,6 +44,15 @@ def test_preview_format(filename: str) - check_file("preview", filename, pyink.Mode(preview=True)) @@ -1412,7 +1432,7 @@ mode = pyink.Mode(preview=True, target_versions={pyink.TargetVersion.PY38}) --- a/tox.ini +++ b/tox.ini -@@ -97,4 +97,4 @@ setenv = PYTHONPATH = {toxinidir}/src +@@ -95,4 +95,4 @@ setenv = PYTHONPATH = {toxinidir}/src skip_install = True commands = pip install -e .[d] @@ -1420,7 +1440,7 @@ + pyink --check {toxinidir}/src {toxinidir}/tests --- a/trans.py +++ b/trans.py -@@ -31,8 +31,8 @@ else: +@@ -27,8 +27,8 @@ from typing import ( from mypy_extensions import trait from pyink.comments import contains_pragma_comment @@ -1431,7 +1451,7 @@ from pyink.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, -@@ -196,9 +196,18 @@ class StringTransformer(ABC): +@@ -192,9 +192,18 @@ class StringTransformer(ABC): # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with # `abc.ABC`. @@ -1451,7 +1471,7 @@ @abstractmethod def do_match(self, line: Line) -> TMatchResult: -@@ -653,7 +662,9 @@ class StringMerger(StringTransformer, Cu +@@ -649,7 +658,9 @@ class StringMerger(StringTransformer, Cu S_leaf = Leaf(token.STRING, S) if self.normalize_strings: @@ -1462,7 +1482,7 @@ # Fill the 'custom_splits' list with the appropriate CustomSplit objects. temp_string = S_leaf.value[len(prefix) + 1 : -1] -@@ -894,7 +905,13 @@ class StringParenStripper(StringTransfor +@@ -890,7 +901,13 @@ class StringParenStripper(StringTransfor idx += 1 if string_indices: @@ -1477,7 +1497,7 @@ return TErr("This line has no strings wrapped in parens.") def do_transform( -@@ -1092,7 +1109,7 @@ class BaseStringSplitter(StringTransform +@@ -1091,7 +1108,7 @@ class BaseStringSplitter(StringTransform # NN: The leaf that is after N. # WMA4 the whitespace at the beginning of the line. @@ -1486,7 +1506,7 @@ if is_valid_index(string_idx - 1): p_idx = string_idx - 1 -@@ -1446,7 +1463,7 @@ class StringSplitter(BaseStringSplitter, +@@ -1445,7 +1462,7 @@ class StringSplitter(BaseStringSplitter, characters expand to two columns). """ result = self.line_length @@ -1495,7 +1515,7 @@ result -= 1 if ends_with_comma else 0 result -= string_op_leaves_length return result -@@ -1457,11 +1474,11 @@ class StringSplitter(BaseStringSplitter, +@@ -1456,11 +1473,11 @@ class StringSplitter(BaseStringSplitter, # The last index of a string of length N is N-1. max_break_width -= 1 # Leading whitespace is not present in the string value (e.g. Leaf.value). @@ -1509,7 +1529,7 @@ ) return -@@ -1758,7 +1775,9 @@ class StringSplitter(BaseStringSplitter, +@@ -1757,7 +1774,9 @@ class StringSplitter(BaseStringSplitter, def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None: if self.normalize_strings: @@ -1520,7 +1540,7 @@ def _normalize_f_string(self, string: str, prefix: str) -> str: """ -@@ -1881,7 +1900,8 @@ class StringParenWrapper(BaseStringSplit +@@ -1880,7 +1899,8 @@ class StringParenWrapper(BaseStringSplit char == " " or char in SPLIT_SAFE_CHARS for char in string_value ): # And will still violate the line length limit when split... @@ -1530,7 +1550,7 @@ if str_width(string_value) > max_string_width: # And has no associated custom splits... if not self.has_custom_splits(string_value): -@@ -2127,7 +2147,7 @@ class StringParenWrapper(BaseStringSplit +@@ -2126,7 +2146,7 @@ class StringParenWrapper(BaseStringSplit string_value = LL[string_idx].value string_line = Line( mode=line.mode, diff --git a/pyproject.toml b/pyproject.toml index 95704402787..fac260fbae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ build-backend = "hatchling.build" name = "pyink" description = "Pyink is a python formatter, forked from Black with slightly different behavior." license = { text = "MIT" } -requires-python = ">=3.7" +requires-python = ">=3.8" readme = "README.md" authors = [{name = "The Pyink Maintainers", email = "pyink-maintainers@google.com"}] classifiers = [ @@ -26,11 +26,11 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance", ] @@ -41,9 +41,8 @@ dependencies = [ "pathspec>=0.9.0", "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", - "typed-ast>=1.4.2; python_version < '3.8' and implementation_name == 'cpython'", - "typing_extensions>=3.10.0.0; python_version < '3.10'", - "black>=23.3.0", + "typing_extensions>=4.0.1; python_version < '3.11'", + "black==23.9.1", ] dynamic = ["version"] @@ -93,10 +92,12 @@ filterwarnings = [ # this is mitigated by a try/catch in https://github.com/psf/black/pull/3198/ # this ignore can be removed when support for aiohttp 3.x is dropped. '''ignore:Middleware decorator is deprecated since 4\.0 and its behaviour is default, you can simply remove this decorator:DeprecationWarning''', - # this is mitigated by https://github.com/python/cpython/issues/79071 in python 3.8+ - # this ignore can be removed when support for 3.7 is dropped. - '''ignore:Bare functions are deprecated, use async ones:DeprecationWarning''', # aiohttp is using deprecated cgi modules - Safe to remove when fixed: # https://github.com/aio-libs/aiohttp/issues/6905 '''ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning''', + # Work around https://github.com/pytest-dev/pytest/issues/10977 for Python 3.12 + '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning''', + # Will be fixed with aiohttp 3.9.0 + # https://github.com/aio-libs/aiohttp/pull/7302 + "ignore:datetime.*utcfromtimestamp\\(\\) is deprecated and scheduled for removal:DeprecationWarning", ] diff --git a/src/pyink/__init__.py b/src/pyink/__init__.py index 43db6529900..b946ab310a6 100644 --- a/src/pyink/__init__.py +++ b/src/pyink/__init__.py @@ -7,7 +7,7 @@ import traceback from contextlib import contextmanager from dataclasses import replace -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from json.decoder import JSONDecodeError from pathlib import Path @@ -35,7 +35,7 @@ from pathspec.patterns.gitwildmatch import GitWildMatchPatternError from _pyink_version import version as __version__ -from pyink.cache import Cache, get_cache_info, read_cache, write_cache +from pyink.cache import Cache from pyink.comments import normalize_fmt_off from pyink.const import ( DEFAULT_EXCLUDES, @@ -64,15 +64,9 @@ ) from pyink.linegen import LN, LineGenerator, transform_line from pyink.lines import EmptyLineTracker, LinesBlock -from pyink.mode import ( - FUTURE_FLAG_TO_FEATURE, - VERSION_TO_FEATURES, - Feature, - Mode, - QuoteStyle, - TargetVersion, - supports_feature, -) +from pyink.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature +from pyink.mode import Mode as Mode # re-exported +from pyink.mode import QuoteStyle, TargetVersion, supports_feature from pyink.nodes import ( STARS, is_number_token, @@ -131,7 +125,9 @@ def read_pyproject_toml( otherwise. """ if not value: - value = find_pyproject_toml(ctx.params.get("src", ())) + value = find_pyproject_toml( + ctx.params.get("src", ()), ctx.params.get("stdin_filename", None) + ) if value is None: return None @@ -159,6 +155,16 @@ def read_pyproject_toml( "target-version", "Config key target-version must be a list" ) + exclude = config.get("exclude") + if exclude is not None and not isinstance(exclude, str): + raise click.BadOptionUsage("exclude", "Config key exclude must be a string") + + extend_exclude = config.get("extend_exclude") + if extend_exclude is not None and not isinstance(extend_exclude, str): + raise click.BadOptionUsage( + "extend-exclude", "Config key extend-exclude must be a string" + ) + default_map: Dict[str, Any] = {} if ctx.default_map: default_map.update(ctx.default_map) @@ -401,6 +407,7 @@ def validate_regex( @click.option( "--stdin-filename", type=str, + is_eager=True, help=( "The name of the file when passing it through stdin. Useful to make " "sure Black will respect --force-exclude option on some " @@ -412,7 +419,10 @@ def validate_regex( "--workers", type=click.IntRange(min=1), default=None, - help="Number of parallel workers [default: number of CPUs in the system]", + help=( + "Number of parallel workers [default: PYINK_NUM_WORKERS environment variable " + "or number of CPUs in the system]" + ), ) @click.option( "-q", @@ -521,26 +531,6 @@ def main( # noqa: C901 fg="blue", ) - normalized = [ - ( - (source, source) - if source == "-" - else (normalize_path_maybe_ignore(Path(source), root), source) - ) - for source in src - ] - srcs_string = ", ".join( - [ - ( - f'"{_norm}"' - if _norm - else f'\033[31m"{source} (skipping - invalid)"\033[34m' - ) - for _norm, source in normalized - ] - ) - out(f"Sources to be formatted: {srcs_string}", fg="blue") - if config: config_source = ctx.get_parameter_source("config") user_level_config = str(find_user_pyproject_toml()) @@ -636,9 +626,10 @@ def main( # noqa: C901 content=code, fast=fast, write_back=write_back, mode=mode, report=report ) else: + assert root is not None # root is only None if code is not None try: sources = get_sources( - ctx=ctx, + root=root, src=src, quiet=quiet, verbose=verbose, @@ -692,7 +683,7 @@ def main( # noqa: C901 def get_sources( *, - ctx: click.Context, + root: Path, src: Tuple[str, ...], quiet: bool, verbose: bool, @@ -705,7 +696,6 @@ def get_sources( ) -> Set[Path]: """Compute the set of files to be formatted.""" sources: Set[Path] = set() - root = ctx.obj["root"] using_default_exclude = exclude is None exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude @@ -721,9 +711,15 @@ def get_sources( is_stdin = False if is_stdin or p.is_file(): - normalized_path = normalize_path_maybe_ignore(p, ctx.obj["root"], report) + normalized_path: Optional[str] = normalize_path_maybe_ignore( + p, root, report + ) if normalized_path is None: + if verbose: + out(f'Skipping invalid source: "{normalized_path}"', fg="red") continue + if verbose: + out(f'Found input source: "{normalized_path}"', fg="blue") normalized_path = "/" + normalized_path # Hard-exclude any files that matches the `--force-exclude` regex. @@ -739,13 +735,18 @@ def get_sources( p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose=verbose, quiet=quiet + warn=verbose or not quiet ): continue sources.add(p) elif p.is_dir(): - p = root / normalize_path_maybe_ignore(p, ctx.obj["root"], report) + p_relative = normalize_path_maybe_ignore(p, root, report) + assert p_relative is not None + p = root / p_relative + if verbose: + out(f'Found input source directory: "{p}"', fg="blue") + if using_default_exclude: gitignore = { root: root_gitignore, @@ -754,7 +755,7 @@ def get_sources( sources.update( gen_python_files( p.iterdir(), - ctx.obj["root"], + root, include, exclude, extend_exclude, @@ -766,9 +767,12 @@ def get_sources( ) ) elif s == "-": + if verbose: + out("Found input source stdin", fg="blue") sources.add(p) else: err(f"invalid path: {s}") + return sources @@ -848,12 +852,9 @@ def reformat_one( ): changed = Changed.YES else: - cache: Cache = {} + cache = Cache.read(mode) if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - cache = read_cache(mode) - res_src = src.resolve() - res_src_s = str(res_src) - if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src): + if not cache.is_changed(src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( src, fast=fast, write_back=write_back, mode=mode, lines=lines @@ -862,7 +863,7 @@ def reformat_one( if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( write_back is WriteBack.CHECK and changed is Changed.NO ): - write_cache(cache, [src], mode) + cache.write([src]) report.done(src, changed) except Exception as exc: if report.verbose: @@ -890,7 +891,7 @@ def format_file_in_place( elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) - then = datetime.utcfromtimestamp(src.stat().st_mtime) + then = datetime.fromtimestamp(src.stat().st_mtime, timezone.utc) header = b"" with open(src, "rb") as buf: if mode.skip_source_first_line: @@ -913,9 +914,9 @@ def format_file_in_place( with open(src, "w", encoding=encoding, newline=newline) as f: f.write(dst_contents) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - now = datetime.utcnow() - src_name = f"{src}\t{then} +0000" - dst_name = f"{src}\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"{src}\t{then}" + dst_name = f"{src}\t{now}" if mode.is_ipynb: diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) else: @@ -954,7 +955,7 @@ def format_stdin_to_stdout( write a diff to stdout. The `mode` argument is passed to :func:`format_file_contents`. """ - then = datetime.utcnow() + then = datetime.now(timezone.utc) if content is None: src, encoding, newline = decode_bytes(sys.stdin.buffer.read()) @@ -979,9 +980,9 @@ def format_stdin_to_stdout( dst += "\n" f.write(dst) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - now = datetime.utcnow() - src_name = f"STDIN\t{then} +0000" - dst_name = f"STDOUT\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"STDIN\t{then}" + dst_name = f"STDOUT\t{now}" d = diff(src, dst, src_name, dst_name) if write_back == WriteBack.COLOR_DIFF: d = color_diff(d) @@ -1396,6 +1397,9 @@ def get_features_used( # noqa: C901 ): features.add(Feature.VARIADIC_GENERICS) + elif n.type in (syms.type_stmt, syms.typeparams): + features.add(Feature.TYPE_PARAMS) + return features @@ -1528,40 +1532,6 @@ def nullcontext() -> Iterator[None]: yield -def patch_click() -> None: - """Make Click not crash on Python 3.6 with LANG=C. - - On certain misconfigured environments, Python 3 selects the ASCII encoding as the - default which restricts paths that it can access during the lifetime of the - application. Click refuses to work in this scenario by raising a RuntimeError. - - In case of Black the likelihood that non-ASCII characters are going to be used in - file paths is minimal since it's Python source code. Moreover, this crash was - spurious on Python 3.7 thanks to PEP 538 and PEP 540. - """ - modules: List[Any] = [] - try: - from click import core - except ImportError: - pass - else: - modules.append(core) - try: - # Removed in Click 8.1.0 and newer; we keep this around for users who have - # older versions installed. - from click import _unicodefun # type: ignore - except ImportError: - pass - else: - modules.append(_unicodefun) - - for module in modules: - if hasattr(module, "_verify_python3_env"): - module._verify_python3_env = lambda: None - if hasattr(module, "_verify_python_env"): - module._verify_python_env = lambda: None - - def patched_main() -> None: # PyInstaller patches multiprocessing to need freeze_support() even in non-Windows # environments so just assume we always need to call it if frozen. @@ -1570,7 +1540,6 @@ def patched_main() -> None: freeze_support() - patch_click() main() diff --git a/src/pyink/_width_table.py b/src/pyink/_width_table.py index 303738ea79e..b19b5e3129b 100644 --- a/src/pyink/_width_table.py +++ b/src/pyink/_width_table.py @@ -1,13 +1,7 @@ # Generated by make_width_table.py # wcwidth 0.2.6 # Unicode 15.0.0 -import sys -from typing import List, Tuple - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, List, Tuple WIDTH_TABLE: Final[Tuple[Tuple[int, int, int], ...]] = ( (0, 0, 0), diff --git a/src/pyink/brackets.py b/src/pyink/brackets.py index 24e45c54711..e5b9e12417f 100644 --- a/src/pyink/brackets.py +++ b/src/pyink/brackets.py @@ -1,13 +1,7 @@ """Builds on top of nodes.py to track brackets.""" -import sys from dataclasses import dataclass, field -from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Dict, Final, Iterable, List, Optional, Sequence, Set, Tuple, Union from pyink.nodes import ( BRACKET, diff --git a/src/pyink/cache.py b/src/pyink/cache.py index 54ece9b7b2a..02dee6bbee1 100644 --- a/src/pyink/cache.py +++ b/src/pyink/cache.py @@ -1,21 +1,28 @@ """Caching of formatted files with feature-based invalidation.""" - +import hashlib import os import pickle +import sys import tempfile +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, Set, Tuple +from typing import Dict, Iterable, NamedTuple, Set, Tuple from platformdirs import user_cache_dir from _pyink_version import version as __version__ from pyink.mode import Mode -# types -Timestamp = float -FileSize = int -CacheInfo = Tuple[Timestamp, FileSize] -Cache = Dict[str, CacheInfo] +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class FileData(NamedTuple): + st_mtime: float + st_size: int + hash: str def get_cache_dir() -> Path: @@ -37,61 +44,98 @@ def get_cache_dir() -> Path: CACHE_DIR = get_cache_dir() -def read_cache(mode: Mode) -> Cache: - """Read the cache if it exists and is well formed. - - If it is not well formed, the call to write_cache later should resolve the issue. - """ - cache_file = get_cache_file(mode) - if not cache_file.exists(): - return {} - - with cache_file.open("rb") as fobj: - try: - cache: Cache = pickle.load(fobj) - except (pickle.UnpicklingError, ValueError, IndexError): - return {} - - return cache - - def get_cache_file(mode: Mode) -> Path: return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle" -def get_cache_info(path: Path) -> CacheInfo: - """Return the information used to check if a file is already formatted or not.""" - stat = path.stat() - return stat.st_mtime, stat.st_size - - -def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: - """Split an iterable of paths in `sources` into two sets. - - The first contains paths of files that modified on disk or are not in the - cache. The other contains paths to non-modified files. - """ - todo, done = set(), set() - for src in sources: - res_src = src.resolve() - if cache.get(str(res_src)) != get_cache_info(res_src): - todo.add(src) - else: - done.add(src) - return todo, done - - -def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None: - """Update the cache file.""" - cache_file = get_cache_file(mode) - try: - CACHE_DIR.mkdir(parents=True, exist_ok=True) - new_cache = { - **cache, - **{str(src.resolve()): get_cache_info(src) for src in sources}, - } - with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f: - pickle.dump(new_cache, f, protocol=4) - os.replace(f.name, cache_file) - except OSError: - pass +@dataclass +class Cache: + mode: Mode + cache_file: Path + file_data: Dict[str, FileData] = field(default_factory=dict) + + @classmethod + def read(cls, mode: Mode) -> Self: + """Read the cache if it exists and is well formed. + + If it is not well formed, the call to write later should + resolve the issue. + """ + cache_file = get_cache_file(mode) + if not cache_file.exists(): + return cls(mode, cache_file) + + with cache_file.open("rb") as fobj: + try: + data: Dict[str, Tuple[float, int, str]] = pickle.load(fobj) + file_data = {k: FileData(*v) for k, v in data.items()} + except (pickle.UnpicklingError, ValueError, IndexError): + return cls(mode, cache_file) + + return cls(mode, cache_file, file_data) + + @staticmethod + def hash_digest(path: Path) -> str: + """Return hash digest for path.""" + + data = path.read_bytes() + return hashlib.sha256(data).hexdigest() + + @staticmethod + def get_file_data(path: Path) -> FileData: + """Return file data for path.""" + + stat = path.stat() + hash = Cache.hash_digest(path) + return FileData(stat.st_mtime, stat.st_size, hash) + + def is_changed(self, source: Path) -> bool: + """Check if source has changed compared to cached version.""" + res_src = source.resolve() + old = self.file_data.get(str(res_src)) + if old is None: + return True + + st = res_src.stat() + if st.st_size != old.st_size: + return True + if int(st.st_mtime) != int(old.st_mtime): + new_hash = Cache.hash_digest(res_src) + if new_hash != old.hash: + return True + return False + + def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: + """Split an iterable of paths in `sources` into two sets. + + The first contains paths of files that modified on disk or are not in the + cache. The other contains paths to non-modified files. + """ + changed: Set[Path] = set() + done: Set[Path] = set() + for src in sources: + if self.is_changed(src): + changed.add(src) + else: + done.add(src) + return changed, done + + def write(self, sources: Iterable[Path]) -> None: + """Update the cache file data and write a new cache file.""" + self.file_data.update( + **{str(src.resolve()): Cache.get_file_data(src) for src in sources} + ) + try: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=str(self.cache_file.parent), delete=False + ) as f: + # We store raw tuples in the cache because pickling NamedTuples + # doesn't work with mypyc on Python 3.8, and because it's faster. + data: Dict[str, Tuple[float, int, str]] = { + k: (*v,) for k, v in self.file_data.items() + } + pickle.dump(data, f, protocol=4) + os.replace(f.name, self.cache_file) + except OSError: + pass diff --git a/src/pyink/comments.py b/src/pyink/comments.py index 3f7fbaea862..839ae630e62 100644 --- a/src/pyink/comments.py +++ b/src/pyink/comments.py @@ -1,13 +1,7 @@ import re -import sys from dataclasses import dataclass from functools import lru_cache -from typing import Iterator, List, Optional, Union - -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final +from typing import Final, Iterator, List, Optional, Union from pyink.nodes import ( CLOSING_BRACKETS, diff --git a/src/pyink/concurrency.py b/src/pyink/concurrency.py index 90baa11d1c3..3a923a1ec34 100644 --- a/src/pyink/concurrency.py +++ b/src/pyink/concurrency.py @@ -17,7 +17,7 @@ from mypy_extensions import mypyc_attr from pyink import WriteBack, format_file_in_place -from pyink.cache import Cache, filter_cached, read_cache, write_cache +from pyink.cache import Cache from pyink.mode import Mode from pyink.output import err from pyink.report import Changed, Report @@ -80,7 +80,8 @@ def reformat_many( executor: Executor if workers is None: - workers = os.cpu_count() or 1 + workers = int(os.environ.get("PYINK_NUM_WORKERS", 0)) + workers = workers or os.cpu_count() or 1 if sys.platform == "win32": # Work around https://bugs.python.org/issue26903 workers = min(workers, 60) @@ -132,10 +133,9 @@ async def schedule_formatting( `write_back`, `fast`, and `mode` options are passed to :func:`format_file_in_place`. """ - cache: Cache = {} + cache = Cache.read(mode) if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - cache = read_cache(mode) - sources, cached = filter_cached(cache, sources) + sources, cached = cache.filtered_cached(sources) for src in sorted(cached): report.done(src, Changed.CACHED) if not sources: @@ -184,4 +184,4 @@ async def schedule_formatting( if cancelled: await asyncio.gather(*cancelled, return_exceptions=True) if sources_to_cache: - write_cache(cache, sources_to_cache, mode) + cache.write(sources_to_cache) diff --git a/src/pyink/const.py b/src/pyink/const.py index 7832ed90fa1..151236fcca1 100644 --- a/src/pyink/const.py +++ b/src/pyink/const.py @@ -1,4 +1,4 @@ DEFAULT_LINE_LENGTH = 88 -DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|\.ipynb_checkpoints|_build|buck-out|build|dist|__pypackages__)/" # noqa: B950 +DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.ipynb_checkpoints|\.mypy_cache|\.nox|\.pytest_cache|\.ruff_cache|\.tox|\.svn|\.venv|\.vscode|__pypackages__|_build|buck-out|build|dist|venv)/" # noqa: B950 DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__PYINK_STDIN_FILENAME__" diff --git a/src/pyink/files.py b/src/pyink/files.py index 1ffd041a621..ae6f2b0a094 100644 --- a/src/pyink/files.py +++ b/src/pyink/files.py @@ -42,7 +42,7 @@ import colorama # noqa: F401 -@lru_cache() +@lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None ) -> Tuple[Path, str]: @@ -89,9 +89,11 @@ def find_project_root( return directory, "file system root" -def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: +def find_pyproject_toml( + path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None +) -> Optional[str]: """Find the absolute filepath to a pyproject.toml if it exists""" - path_project_root, _ = find_project_root(path_search_start) + path_project_root, _ = find_project_root(path_search_start, stdin_filename) path_pyproject_toml = path_project_root / "pyproject.toml" if path_pyproject_toml.is_file(): return str(path_pyproject_toml) @@ -210,7 +212,7 @@ def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet: return SpecifierSet(",".join(str(s) for s in specifiers)) -@lru_cache() +@lru_cache def find_user_pyproject_toml() -> Path: r"""Return the path to the top-level user configuration for pyink. @@ -230,7 +232,7 @@ def find_user_pyproject_toml() -> Path: return user_config_path.resolve() -@lru_cache() +@lru_cache def get_gitignore(root: Path) -> PathSpec: """Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" @@ -274,15 +276,24 @@ def normalize_path_maybe_ignore( return root_relative_path -def path_is_ignored( - path: Path, gitignore_dict: Dict[Path, PathSpec], report: Report +def _path_is_ignored( + root_relative_path: str, + root: Path, + gitignore_dict: Dict[Path, PathSpec], + report: Report, ) -> bool: + path = root / root_relative_path + # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must + # ensure that gitignore_dict is ordered from least specific to most specific. for gitignore_path, pattern in gitignore_dict.items(): - relative_path = normalize_path_maybe_ignore(path, gitignore_path, report) - if relative_path is None: + try: + relative_path = path.relative_to(gitignore_path).as_posix() + except ValueError: break if pattern.match_file(relative_path): - report.path_ignored(path, "matches a .gitignore file content") + report.path_ignored( + path.relative_to(root), "matches a .gitignore file content" + ) return True return False @@ -319,33 +330,37 @@ def gen_python_files( assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in paths: - normalized_path = normalize_path_maybe_ignore(child, root, report) - if normalized_path is None: - continue + root_relative_path = child.absolute().relative_to(root).as_posix() # First ignore files matching .gitignore, if passed - if gitignore_dict and path_is_ignored(child, gitignore_dict, report): + if gitignore_dict and _path_is_ignored( + root_relative_path, root, gitignore_dict, report + ): continue # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. - normalized_path = "/" + normalized_path + root_relative_path = "/" + root_relative_path if child.is_dir(): - normalized_path += "/" + root_relative_path += "/" - if path_is_excluded(normalized_path, exclude): + if path_is_excluded(root_relative_path, exclude): report.path_ignored(child, "matches the --exclude regular expression") continue - if path_is_excluded(normalized_path, extend_exclude): + if path_is_excluded(root_relative_path, extend_exclude): report.path_ignored( child, "matches the --extend-exclude regular expression" ) continue - if path_is_excluded(normalized_path, force_exclude): + if path_is_excluded(root_relative_path, force_exclude): report.path_ignored(child, "matches the --force-exclude regular expression") continue + normalized_path = normalize_path_maybe_ignore(child, root, report) + if normalized_path is None: + continue + if child.is_dir(): # If gitignore is None, gitignore usage is disabled, while a Falsey # gitignore is when the directory doesn't have a .gitignore file. @@ -371,7 +386,7 @@ def gen_python_files( elif child.is_file(): if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose=verbose, quiet=quiet + warn=verbose or not quiet ): continue include_match = include.search(normalized_path) if include else True diff --git a/src/pyink/handle_ipynb_magics.py b/src/pyink/handle_ipynb_magics.py index 915a1e760e8..9dbfa8fb699 100644 --- a/src/pyink/handle_ipynb_magics.py +++ b/src/pyink/handle_ipynb_magics.py @@ -6,6 +6,7 @@ import secrets import sys from functools import lru_cache +from importlib.util import find_spec from typing import Dict, List, Optional, Tuple if sys.version_info >= (3, 10): @@ -55,21 +56,18 @@ class Replacement: src: str -@lru_cache() -def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: - try: - import IPython # noqa:F401 - import tokenize_rt # noqa:F401 - except ModuleNotFoundError: - if verbose or not quiet: - msg = ( - "Skipping .ipynb files as Jupyter dependencies are not installed.\n" - 'You can fix this by running ``pip install "black[jupyter]"``' - ) - out(msg) - return False - else: - return True +@lru_cache +def jupyter_dependencies_are_installed(*, warn: bool) -> bool: + installed = ( + find_spec("tokenize_rt") is not None and find_spec("IPython") is not None + ) + if not installed and warn: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + 'You can fix this by running ``pip install "black[jupyter]"``' + ) + out(msg) + return installed def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: @@ -330,7 +328,8 @@ class CellMagicFinder(ast.NodeVisitor): For example, - %%time\nfoo() + %%time\n + foo() would have been transformed to diff --git a/src/pyink/linegen.py b/src/pyink/linegen.py index b9a998d33ea..43ed13b23b1 100644 --- a/src/pyink/linegen.py +++ b/src/pyink/linegen.py @@ -55,6 +55,7 @@ is_stub_body, is_stub_suite, is_tuple_containing_walrus, + is_type_ignore_comment_string, is_vararg, is_walrus_assignment, is_yield, @@ -246,6 +247,18 @@ def visit_stmt( yield from self.visit(child) + def visit_typeparams(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[0].prefix = "" + + def visit_typevartuple(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + + def visit_paramspec(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if Preview.wrap_long_dict_values_in_parens in self.mode: for i, child in enumerate(node.children): @@ -299,7 +312,9 @@ def visit_match_case(self, node: Node) -> Iterator[Line]: def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" - if self.mode.is_pyi and is_stub_suite(node): + if ( + self.mode.is_pyi or Preview.dummy_implementations in self.mode + ) and is_stub_suite(node): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -314,7 +329,9 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: is_suite_like = node.parent and node.parent.type in STATEMENT if is_suite_like: - if self.mode.is_pyi and is_stub_body(node): + if ( + self.mode.is_pyi or Preview.dummy_implementations in self.mode + ) and is_stub_body(node): yield from self.visit_default(node) else: yield from self.line(Indentation.SCOPE) @@ -323,7 +340,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: else: if ( - not self.mode.is_pyi + not (self.mode.is_pyi or Preview.dummy_implementations in self.mode) or not node.parent or not is_stub_suite(node.parent) ): @@ -1015,6 +1032,13 @@ def bracket_split_build_line( and leaves[0].parent.next_sibling and leaves[0].parent.next_sibling.type == token.VBAR ) + # Except the false negatives above for PEP 604 unions where we + # can't add the comma. + and not ( + leaves[0].parent + and leaves[0].parent.next_sibling + and leaves[0].parent.next_sibling.type == token.VBAR + ) ) if original.is_import or no_commas: @@ -1489,8 +1513,13 @@ def maybe_make_parens_invisible_in_atom( if is_lpar_token(first) and is_rpar_token(last): middle = node.children[1] # make parentheses invisible - first.value = "" - last.value = "" + if ( + # If the prefix of `middle` includes a type comment with + # ignore annotation, then we do not remove the parentheses + not is_type_ignore_comment_string(middle.prefix.strip()) + ): + first.value = "" + last.value = "" maybe_make_parens_invisible_in_atom( middle, parent=parent, diff --git a/src/pyink/lines.py b/src/pyink/lines.py index 0fd70d2f432..c38f944b396 100644 --- a/src/pyink/lines.py +++ b/src/pyink/lines.py @@ -30,6 +30,7 @@ is_multiline_string, is_one_sequence_between, is_type_comment, + is_type_ignore_comment, is_with_or_async_with_stmt, replace_child, syms, @@ -65,7 +66,7 @@ def num_spaces(self, mode: Mode) -> int: class Line: """Holds leaves and comments. Can be printed with `str(line)`.""" - mode: Mode + mode: Mode = field(repr=False) depth: Tuple[Indentation, ...] = field(default_factory=tuple) leaves: List[Leaf] = field(default_factory=list) # keys ordered like `leaves` @@ -100,7 +101,9 @@ def append( # Note: at this point leaf.prefix should be empty except for # imports, for which we only preserve newlines. leaf.prefix += whitespace( - leaf, complex_subscript=self.is_complex_subscript(leaf) + leaf, + complex_subscript=self.is_complex_subscript(leaf), + mode=self.mode, ) if self.inside_brackets or not preformatted or track_bracket: self.bracket_tracker.mark(leaf) @@ -184,6 +187,13 @@ def is_def(self) -> bool: and second_leaf.value == "def" ) + @property + def is_stub_def(self) -> bool: + """Is this line a function definition with a body consisting only of "..."?""" + return self.is_def and self.leaves[-4:] == [Leaf(token.COLON, ":")] + [ + Leaf(token.DOT, ".") for _ in range(3) + ] + @property def is_class_paren_empty(self) -> bool: """Is this a class with no base classes but using parentheses? @@ -271,7 +281,7 @@ def contains_uncollapsable_type_comments(self) -> bool: for comment in comments: if is_type_comment(comment): if comment_seen or ( - not is_type_comment(comment, " ignore") + not is_type_ignore_comment(comment) and leaf_id not in ignored_ids ): return True @@ -308,7 +318,7 @@ def contains_unsplittable_type_ignore(self) -> bool: # line. for node in self.leaves[-2:]: for comment in self.comments.get(id(node), []): - if is_type_comment(comment, " ignore"): + if is_type_ignore_comment(comment): return True return False @@ -611,17 +621,24 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: first_leaf.prefix = "" else: before = 0 + + user_had_newline = bool(before) depth = current_line.depth + + previous_def = None while self.previous_defs and len(self.previous_defs[-1].depth) >= len(depth): + previous_def = self.previous_defs.pop() + + if previous_def is not None: + assert self.previous_line is not None if self.mode.is_pyi: - assert self.previous_line is not None if depth and not current_line.is_def and self.previous_line.is_def: # Empty lines between attributes and methods should be preserved. - before = min(1, before) + before = 1 if user_had_newline else 0 elif ( Preview.blank_line_after_nested_stub_class in self.mode - and self.previous_defs[-1].is_class - and not self.previous_defs[-1].is_stub_class + and previous_def.is_class + and not previous_def.is_stub_class ): before = 1 elif depth: @@ -633,7 +650,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 1 elif ( not depth - and self.previous_defs[-1].depth + and previous_def.depth and current_line.leaves[-1].type == token.COLON and ( current_line.leaves[0].value @@ -650,9 +667,11 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 1 else: before = 2 - self.previous_defs.pop() + if current_line.is_decorator or current_line.is_def or current_line.is_class: - return self._maybe_empty_lines_for_class_or_def(current_line, before) + return self._maybe_empty_lines_for_class_or_def( + current_line, before, user_had_newline + ) if ( self.previous_line @@ -683,6 +702,8 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: and self.previous_line.is_class and current_line.is_triple_quoted_string ): + if Preview.no_blank_line_before_class_docstring in current_line.mode: + return 0, 1 return before, 1 if ( @@ -697,8 +718,8 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: return 0, 0 return before, 0 - def _maybe_empty_lines_for_class_or_def( - self, current_line: Line, before: int + def _maybe_empty_lines_for_class_or_def( # noqa: C901 + self, current_line: Line, before: int, user_had_newline: bool ) -> Tuple[int, int]: if not current_line.is_decorator: self.previous_defs.append(current_line) @@ -748,6 +769,17 @@ def _maybe_empty_lines_for_class_or_def( newlines = 0 else: newlines = 1 + # Remove case `self.previous_line.depth > current_line.depth` below when + # this becomes stable. + # + # Don't inspect the previous line if it's part of the body of the previous + # statement in the same level, we always want a blank line if there's + # something with a body preceding. + elif ( + Preview.blank_line_between_nested_and_def_stub_file in current_line.mode + and self.previous_line.depth > current_line.depth + ): + newlines = 1 elif ( current_line.is_def or current_line.is_decorator ) and not self.previous_line.is_def: @@ -765,6 +797,14 @@ def _maybe_empty_lines_for_class_or_def( newlines = 0 else: newlines = 1 if current_line.depth else 2 + # If a user has left no space after a dummy implementation, don't insert + # new lines. This is useful for instance for @overload or Protocols. + if ( + Preview.dummy_implementations in self.mode + and self.previous_line.is_stub_def + and not user_had_newline + ): + newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block if previous_block is not None: diff --git a/src/pyink/mode.py b/src/pyink/mode.py index ff962938359..76db49da4bc 100644 --- a/src/pyink/mode.py +++ b/src/pyink/mode.py @@ -4,19 +4,13 @@ chosen by the user. """ -import sys from dataclasses import dataclass, field from enum import Enum, auto from hashlib import sha256 from operator import attrgetter -from typing import Dict, Set +from typing import Dict, Final, Literal, Set from warnings import warn -if sys.version_info < (3, 8): - from typing_extensions import Final, Literal -else: - from typing import Final, Literal - from pyink.const import DEFAULT_LINE_LENGTH @@ -30,6 +24,7 @@ class TargetVersion(Enum): PY39 = 9 PY310 = 10 PY311 = 11 + PY312 = 12 class Feature(Enum): @@ -51,6 +46,7 @@ class Feature(Enum): VARIADIC_GENERICS = 15 DEBUG_F_STRINGS = 16 PARENTHESIZED_CONTEXT_MANAGERS = 17 + TYPE_PARAMS = 18 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -143,6 +139,25 @@ class Feature(Enum): Feature.EXCEPT_STAR, Feature.VARIADIC_GENERICS, }, + TargetVersion.PY312: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + }, } @@ -155,9 +170,11 @@ class Preview(Enum): add_trailing_comma_consistently = auto() blank_line_after_nested_stub_class = auto() + blank_line_between_nested_and_def_stub_file = auto() hex_codes_in_unicode_sequences = auto() improved_async_statements_handling = auto() multiline_string_handling = auto() + no_blank_line_before_class_docstring = auto() prefer_splitting_right_hand_side_of_assignments = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. @@ -166,6 +183,8 @@ class Preview(Enum): skip_magic_trailing_comma_in_subscript = auto() wrap_long_dict_values_in_parens = auto() wrap_multiple_context_managers_in_parens = auto() + dummy_implementations = auto() + walrus_subscript = auto() class Deprecated(UserWarning): @@ -226,6 +245,9 @@ def __contains__(self, feature: Preview) -> bool: """ if feature is Preview.string_processing: return self.preview or self.experimental_string_processing + # dummy_implementations is temporarily disabled in Pyink. + if feature is Preview.dummy_implementations and self.is_pyink: + return False return self.preview def get_cache_key(self) -> str: diff --git a/src/pyink/nodes.py b/src/pyink/nodes.py index 4755d7803cb..537e586b461 100644 --- a/src/pyink/nodes.py +++ b/src/pyink/nodes.py @@ -3,12 +3,8 @@ """ import sys -from typing import Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union +from typing import Final, Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final if sys.version_info >= (3, 10): from typing import TypeGuard else: @@ -17,6 +13,7 @@ from mypy_extensions import mypyc_attr from pyink.cache import CACHE_DIR +from pyink.mode import Mode, Preview from pyink.strings import has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token @@ -175,7 +172,7 @@ def visit_default(self, node: LN) -> Iterator[T]: yield from self.visit(child) -def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 +def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # noqa: C901 """Return whitespace prefix if needed for the given `leaf`. `complex_subscript` signals whether the given leaf is part of a subscription @@ -349,6 +346,11 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 return NO + elif Preview.walrus_subscript in mode and ( + t == token.COLONEQUAL or prev.type == token.COLONEQUAL + ): + return SPACE + elif not complex_subscript: return NO @@ -728,6 +730,11 @@ def is_multiline_string(leaf: Leaf) -> bool: def is_stub_suite(node: Node) -> bool: """Return True if `node` is a suite with a stub body.""" + + # If there is a comment, we want to keep it. + if node.prefix.strip(): + return False + if ( len(node.children) != 4 or node.children[0].type != token.NEWLINE @@ -736,6 +743,9 @@ def is_stub_suite(node: Node) -> bool: ): return False + if node.children[3].prefix.strip(): + return False + return is_stub_body(node.children[2]) @@ -749,7 +759,8 @@ def is_stub_body(node: LN) -> bool: child = node.children[0] return ( - child.type == syms.atom + not child.prefix.strip() + and child.type == syms.atom and len(child.children) == 3 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children) ) @@ -826,12 +837,27 @@ def is_async_stmt_or_funcdef(leaf: Leaf) -> bool: ) -def is_type_comment(leaf: Leaf, suffix: str = "") -> bool: - """Return True if the given leaf is a special comment. - Only returns true for type comments for now.""" +def is_type_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment. This function should only + be used for general type comments (excluding ignore annotations, which should + use `is_type_ignore_comment`). Note that general type comments are no longer + used in modern version of Python, this function may be deprecated in the future.""" t = leaf.type v = leaf.value - return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix) + return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") + + +def is_type_ignore_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment with ignore annotation.""" + t = leaf.type + v = leaf.value + return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v) + + +def is_type_ignore_comment_string(value: str) -> bool: + """Return True if the given string match with type comment with + ignore annotation.""" + return value.startswith("# type: ignore") def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: diff --git a/src/pyink/parsing.py b/src/pyink/parsing.py index 404e9a10791..79174b092bf 100644 --- a/src/pyink/parsing.py +++ b/src/pyink/parsing.py @@ -2,14 +2,8 @@ Parse Python code and perform AST validation. """ import ast -import platform import sys -from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, Iterable, Iterator, List, Set, Tuple from pyink.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature from pyink.nodes import syms @@ -20,25 +14,6 @@ from blib2to3.pgen2.tokenize import TokenError from blib2to3.pytree import Leaf, Node -ast3: Any - -_IS_PYPY = platform.python_implementation() == "PyPy" - -try: - from typed_ast import ast3 -except ImportError: - if sys.version_info < (3, 8) and not _IS_PYPY: - print( - "The typed_ast package is required but not installed.\n" - "You can upgrade to Python 3.8+ or install typed_ast with\n" - "`python3 -m pip install typed-ast`.", - file=sys.stderr, - ) - sys.exit(1) - else: - ast3 = ast - - PY2_HINT: Final = "Python 2 support was removed in version 22.0." @@ -147,31 +122,14 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( src: str, version: Tuple[int, int], *, type_comments: bool -) -> Union[ast.AST, ast3.AST]: +) -> ast.AST: filename = "" - # typed-ast is needed because of feature version limitations in the builtin ast 3.8> - if sys.version_info >= (3, 8) and version >= (3,): - return ast.parse( - src, filename, feature_version=version, type_comments=type_comments - ) - - if _IS_PYPY: - # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's - # not much we can do as typed-ast won't work either. - if sys.version_info >= (3, 8): - return ast3.parse(src, filename, type_comments=type_comments) - else: - return ast3.parse(src, filename) - else: - if type_comments: - # Typed-ast is guaranteed to be used here and automatically tracks type - # comments separately. - return ast3.parse(src, filename, feature_version=version[1]) - else: - return ast.parse(src, filename) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) -def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: +def parse_ast(src: str) -> ast.AST: # TODO: support Python 4+ ;) versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)] @@ -193,9 +151,6 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: raise SyntaxError(first_error) -ast3_AST: Final[Type[ast3.AST]] = ast3.AST - - def _normalize(lineend: str, value: str) -> str: # To normalize, we strip any leading and trailing space from # each line... @@ -206,23 +161,25 @@ def _normalize(lineend: str, value: str) -> str: return normalized.strip() -def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]: +def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: """Simple visitor generating strings to compare ASTs by content.""" - node = fixup_ast_constants(node) + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.kind == "u" + ): + # It's a quirk of history that we strip the u prefix over here. We used to + # rewrite the AST nodes for Python version compatibility and we never copied + # over the kind + node.kind = None yield f"{' ' * depth}{node.__class__.__name__}(" - type_ignore_classes: Tuple[Type[Any], ...] for field in sorted(node._fields): # noqa: F402 - # TypeIgnore will not be present using pypy < 3.8, so need for this - if not (_IS_PYPY and sys.version_info < (3, 8)): - # TypeIgnore has only one field 'lineno' which breaks this comparison - type_ignore_classes = (ast3.TypeIgnore,) - if sys.version_info >= (3, 8): - type_ignore_classes += (ast.TypeIgnore,) - if isinstance(node, type_ignore_classes): - break + # TypeIgnore has only one field 'lineno' which breaks this comparison + if isinstance(node, ast.TypeIgnore): + break try: value: object = getattr(node, field) @@ -237,51 +194,34 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st # parentheses and they change the AST. if ( field == "targets" - and isinstance(node, (ast.Delete, ast3.Delete)) - and isinstance(item, (ast.Tuple, ast3.Tuple)) + and isinstance(node, ast.Delete) + and isinstance(item, ast.Tuple) ): for elt in item.elts: yield from stringify_ast(elt, depth + 2) - elif isinstance(item, (ast.AST, ast3.AST)): + elif isinstance(item, ast.AST): yield from stringify_ast(item, depth + 2) - # Note that we are referencing the typed-ast ASTs via global variables and not - # direct module attribute accesses because that breaks mypyc. It's probably - # something to do with the ast3 variables being marked as Any leading - # mypy to think this branch is always taken, leaving the rest of the code - # unanalyzed. Tighting up the types for the typed-ast AST types avoids the - # mypyc crash. - elif isinstance(value, (ast.AST, ast3_AST)): + elif isinstance(value, ast.AST): yield from stringify_ast(value, depth + 2) else: normalized: object - # Constant strings may be indented across newlines, if they are - # docstrings; fold spaces after newlines when comparing. Similarly, - # trailing and leading space may be removed. if ( isinstance(node, ast.Constant) and field == "value" and isinstance(value, str) ): + # Constant strings may be indented across newlines, if they are + # docstrings; fold spaces after newlines when comparing. Similarly, + # trailing and leading space may be removed. normalized = _normalize("\n", value) + elif field == "type_comment" and isinstance(value, str): + # Trailing whitespace in type comments is removed. + normalized = value.rstrip() else: normalized = value yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" yield f"{' ' * depth}) # /{node.__class__.__name__}" - - -def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]: - """Map ast nodes deprecated in 3.8 to Constant.""" - if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)): - return ast.Constant(value=node.s) - - if isinstance(node, (ast.Num, ast3.Num)): - return ast.Constant(value=node.n) - - if isinstance(node, (ast.NameConstant, ast3.NameConstant)): - return ast.Constant(value=node.value) - - return node diff --git a/src/pyink/strings.py b/src/pyink/strings.py index ba83c2ba786..7b3d5b5d96e 100644 --- a/src/pyink/strings.py +++ b/src/pyink/strings.py @@ -5,17 +5,11 @@ import re import sys from functools import lru_cache -from typing import List, Match, Pattern - -from blib2to3.pytree import Leaf - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, List, Match, Pattern from pyink._width_table import WIDTH_TABLE from pyink.mode import Quote +from blib2to3.pytree import Leaf STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. STRING_PREFIX_RE: Final = re.compile( diff --git a/src/pyink/trans.py b/src/pyink/trans.py index a4722574e82..b1aaa851146 100644 --- a/src/pyink/trans.py +++ b/src/pyink/trans.py @@ -2,7 +2,6 @@ String transformers that can split and merge strings. """ import re -import sys from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass @@ -12,9 +11,11 @@ ClassVar, Collection, Dict, + Final, Iterable, Iterator, List, + Literal, Optional, Sequence, Set, @@ -23,11 +24,6 @@ Union, ) -if sys.version_info < (3, 8): - from typing_extensions import Final, Literal -else: - from typing import Literal, Final - from mypy_extensions import trait from pyink.comments import contains_pragma_comment @@ -214,11 +210,11 @@ def do_match(self, line: Line) -> TMatchResult: """ Returns: * Ok(string_indices) such that for each index, `line.leaves[index]` - is our target string if a match was able to be made. For - transformers that don't result in more lines (e.g. StringMerger, - StringParenStripper), multiple matches and transforms are done at - once to reduce the complexity. - OR + is our target string if a match was able to be made. For + transformers that don't result in more lines (e.g. StringMerger, + StringParenStripper), multiple matches and transforms are done at + once to reduce the complexity. + OR * Err(CannotTransform), if no match could be made. """ @@ -229,12 +225,12 @@ def do_transform( """ Yields: * Ok(new_line) where new_line is the new transformed line. - OR + OR * Err(CannotTransform) if the transformation failed for some reason. The - `do_match(...)` template method should usually be used to reject - the form of the given Line, but in some cases it is difficult to - know whether or not a Line meets the StringTransformer's - requirements until the transformation is already midway. + `do_match(...)` template method should usually be used to reject + the form of the given Line, but in some cases it is difficult to + know whether or not a Line meets the StringTransformer's + requirements until the transformation is already midway. Side Effects: This method should NOT mutate @line directly, but it MAY mutate the @@ -344,8 +340,8 @@ def pop_custom_splits(self, string: str) -> List[CustomSplit]: Returns: * A list of the custom splits that are mapped to @string, if any - exist. - OR + exist. + OR * [], otherwise. Side Effects: @@ -374,14 +370,14 @@ class StringMerger(StringTransformer, CustomSplitMapMixin): Requirements: (A) The line contains adjacent strings such that ALL of the validation checks listed in StringMerger._validate_msg(...)'s docstring pass. - OR + OR (B) The line contains a string which uses line continuation backslashes. Transformations: Depending on which of the two requirements above where met, either: (A) The string group associated with the target string is merged. - OR + OR (B) All line-continuation backslashes are removed from the target string. Collaborations: @@ -982,17 +978,20 @@ class BaseStringSplitter(StringTransformer): Requirements: * The target string value is responsible for the line going over the - line length limit. It follows that after all of black's other line - split methods have been exhausted, this line (or one of the resulting - lines after all line splits are performed) would still be over the - line_length limit unless we split this string. - AND + line length limit. It follows that after all of black's other line + split methods have been exhausted, this line (or one of the resulting + lines after all line splits are performed) would still be over the + line_length limit unless we split this string. + AND + * The target string is NOT a "pointless" string (i.e. a string that has - no parent or siblings). - AND + no parent or siblings). + AND + * The target string is not followed by an inline comment that appears - to be a pragma. - AND + to be a pragma. + AND + * The target string is not a multiline (i.e. triple-quote) string. """ @@ -1044,7 +1043,7 @@ def _validate(self, line: Line, string_idx: int) -> TResult[None]: Returns: * Ok(None), if ALL of the requirements are met. - OR + OR * Err(CannotTransform), if ANY of the requirements are NOT met. """ LL = line.leaves @@ -1316,9 +1315,9 @@ class StringSplitter(BaseStringSplitter, CustomSplitMapMixin): Requirements: * The line consists ONLY of a single string (possibly prefixed by a - string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE - a trailing comma. - AND + string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE + a trailing comma. + AND * All of the requirements listed in BaseStringSplitter's docstring. Transformations: @@ -1827,26 +1826,26 @@ class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin): addition to the requirements listed below: * The line is a return/yield statement, which returns/yields a string. - OR + OR * The line is part of a ternary expression (e.g. `x = y if cond else - z`) such that the line starts with `else `, where is - some string. - OR + z`) such that the line starts with `else `, where is + some string. + OR * The line is an assert statement, which ends with a string. - OR + OR * The line is an assignment statement (e.g. `x = ` or `x += - `) such that the variable is being assigned the value of some - string. - OR + `) such that the variable is being assigned the value of some + string. + OR * The line is a dictionary key assignment where some valid key is being - assigned the value of some string. - OR + assigned the value of some string. + OR * The line is an lambda expression and the value is a string. - OR + OR * The line starts with an "atom" string that prefers to be wrapped in - parens. It's preferred to be wrapped when it's is an immediate child of - a list/set/tuple literal, AND the string is surrounded by commas (or is - the first/last child). + parens. It's preferred to be wrapped when it's is an immediate child of + a list/set/tuple literal, AND the string is surrounded by commas (or is + the first/last child). Transformations: The chosen string is wrapped in parentheses and then split at the LPAR. @@ -2293,7 +2292,7 @@ def parse(self, leaves: List[Leaf], string_idx: int) -> int: Returns: The index directly after the last leaf which is apart of the string trailer, if a "trailer" exists. - OR + OR @string_idx + 1, if no string "trailer" exists. """ assert leaves[string_idx].type == token.STRING @@ -2307,11 +2306,11 @@ def _next_state(self, leaf: Leaf) -> bool: """ Pre-conditions: * On the first call to this function, @leaf MUST be the leaf that - was directly after the string leaf in question (e.g. if our target - string is `line.leaves[i]` then the first call to this method must - be `line.leaves[i + 1]`). + was directly after the string leaf in question (e.g. if our target + string is `line.leaves[i]` then the first call to this method must + be `line.leaves[i + 1]`). * On the next call to this function, the leaf parameter passed in - MUST be the leaf directly following @leaf. + MUST be the leaf directly following @leaf. Returns: True iff @leaf is apart of the string's trailer. diff --git a/test_requirements.txt b/test_requirements.txt index ef61a1210ee..a3d262bc53d 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,6 @@ coverage >= 5.3 pre-commit pytest >= 6.1.1 -pytest-xdist >= 2.2.1, < 3.0.2 -pytest-cov >= 2.11.1 +pytest-xdist >= 3.0.2 +pytest-cov >= 4.1.0 tox diff --git a/tests/data/miscellaneous/force_py36.py b/tests/data/miscellaneous/force_py36.py index cad935e525a..4c9b70336e7 100644 --- a/tests/data/miscellaneous/force_py36.py +++ b/tests/data/miscellaneous/force_py36.py @@ -1,6 +1,6 @@ # The input source must not contain any Py36-specific syntax (e.g. argument type # annotations, trailing comma after *rest) or this test becomes invalid. -def long_function_name(argument_one, argument_two, argument_three, argument_four, argument_five, argument_six, *rest): ... +def long_function_name(argument_one, argument_two, argument_three, argument_four, argument_five, argument_six, *rest): pass # output # The input source must not contain any Py36-specific syntax (e.g. argument type # annotations, trailing comma after *rest) or this test becomes invalid. @@ -13,4 +13,4 @@ def long_function_name( argument_six, *rest, ): - ... + pass diff --git a/tests/data/miscellaneous/nested_class_stub.pyi b/tests/data/miscellaneous/nested_class_stub.pyi deleted file mode 100644 index daf281b517b..00000000000 --- a/tests/data/miscellaneous/nested_class_stub.pyi +++ /dev/null @@ -1,16 +0,0 @@ -class Outer: - class InnerStub: ... - outer_attr_after_inner_stub: int - class Inner: - inner_attr: int - outer_attr: int - -# output -class Outer: - class InnerStub: ... - outer_attr_after_inner_stub: int - - class Inner: - inner_attr: int - - outer_attr: int diff --git a/tests/data/miscellaneous/nested_stub.pyi b/tests/data/miscellaneous/nested_stub.pyi new file mode 100644 index 00000000000..15e69d854db --- /dev/null +++ b/tests/data/miscellaneous/nested_stub.pyi @@ -0,0 +1,43 @@ +import sys + +class Outer: + class InnerStub: ... + outer_attr_after_inner_stub: int + class Inner: + inner_attr: int + outer_attr: int + +if sys.version_info > (3, 7): + if sys.platform == "win32": + assignment = 1 + def function_definition(self): ... + def f1(self) -> str: ... + if sys.platform != "win32": + def function_definition(self): ... + assignment = 1 + def f2(self) -> str: ... + +# output + +import sys + +class Outer: + class InnerStub: ... + outer_attr_after_inner_stub: int + + class Inner: + inner_attr: int + + outer_attr: int + +if sys.version_info > (3, 7): + if sys.platform == "win32": + assignment = 1 + def function_definition(self): ... + + def f1(self) -> str: ... + if sys.platform != "win32": + def function_definition(self): ... + assignment = 1 + + def f2(self) -> str: ... \ No newline at end of file diff --git a/tests/data/preview/comments7.py b/tests/data/preview/comments7.py index ec2dc501d8e..8b1224017e5 100644 --- a/tests/data/preview/comments7.py +++ b/tests/data/preview/comments7.py @@ -278,8 +278,7 @@ class C: ) def test_fails_invalid_post_data( self, pyramid_config, db_request, post_data, message - ): - ... + ): ... square = Square(4) # type: Optional[Square] diff --git a/tests/data/preview/dummy_implementations.py b/tests/data/preview/dummy_implementations.py new file mode 100644 index 00000000000..e07c25ed129 --- /dev/null +++ b/tests/data/preview/dummy_implementations.py @@ -0,0 +1,99 @@ +from typing import NoReturn, Protocol, Union, overload + + +def dummy(a): ... +def other(b): ... + + +@overload +def a(arg: int) -> int: ... +@overload +def a(arg: str) -> str: ... +@overload +def a(arg: object) -> NoReturn: ... +def a(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + +class Proto(Protocol): + def foo(self, a: int) -> int: + ... + + def bar(self, b: str) -> str: ... + def baz(self, c: bytes) -> str: + ... + + +def dummy_two(): + ... +@dummy +def dummy_three(): + ... + +def dummy_four(): + ... + +@overload +def b(arg: int) -> int: ... + +@overload +def b(arg: str) -> str: ... +@overload +def b(arg: object) -> NoReturn: ... + +def b(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + +# output + +from typing import NoReturn, Protocol, Union, overload + + +def dummy(a): ... +def other(b): ... + + +@overload +def a(arg: int) -> int: ... +@overload +def a(arg: str) -> str: ... +@overload +def a(arg: object) -> NoReturn: ... +def a(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + + +class Proto(Protocol): + def foo(self, a: int) -> int: ... + + def bar(self, b: str) -> str: ... + def baz(self, c: bytes) -> str: ... + + +def dummy_two(): ... +@dummy +def dummy_three(): ... + + +def dummy_four(): ... + + +@overload +def b(arg: int) -> int: ... + + +@overload +def b(arg: str) -> str: ... +@overload +def b(arg: object) -> NoReturn: ... + + +def b(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg diff --git a/tests/data/preview/no_blank_line_before_docstring.py b/tests/data/preview/no_blank_line_before_docstring.py new file mode 100644 index 00000000000..a37362de100 --- /dev/null +++ b/tests/data/preview/no_blank_line_before_docstring.py @@ -0,0 +1,58 @@ +def line_before_docstring(): + + """Please move me up""" + + +class LineBeforeDocstring: + + """Please move me up""" + + +class EvenIfThereIsAMethodAfter: + + """I'm the docstring""" + def method(self): + pass + + +class TwoLinesBeforeDocstring: + + + """I want to be treated the same as if I were closer""" + + +class MultilineDocstringsAsWell: + + """I'm so far + + and on so many lines... + """ + + +# output + + +def line_before_docstring(): + """Please move me up""" + + +class LineBeforeDocstring: + """Please move me up""" + + +class EvenIfThereIsAMethodAfter: + """I'm the docstring""" + + def method(self): + pass + + +class TwoLinesBeforeDocstring: + """I want to be treated the same as if I were closer""" + + +class MultilineDocstringsAsWell: + """I'm so far + + and on so many lines... + """ diff --git a/tests/data/preview/pep_572.py b/tests/data/preview/pep_572.py new file mode 100644 index 00000000000..a50e130ad9c --- /dev/null +++ b/tests/data/preview/pep_572.py @@ -0,0 +1,6 @@ +x[(a:=0):] +x[:(a:=0)] + +# output +x[(a := 0):] +x[:(a := 0)] diff --git a/tests/data/preview_py_310/pep_572.py b/tests/data/preview_py_310/pep_572.py new file mode 100644 index 00000000000..78d4e9e4506 --- /dev/null +++ b/tests/data/preview_py_310/pep_572.py @@ -0,0 +1,12 @@ +x[a:=0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] +x[a:=0,b:=1] + +# output +x[a := 0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] +x[a := 0, b := 1] diff --git a/tests/data/py_312/type_aliases.py b/tests/data/py_312/type_aliases.py new file mode 100644 index 00000000000..84e07e50fe2 --- /dev/null +++ b/tests/data/py_312/type_aliases.py @@ -0,0 +1,13 @@ +type A=int +type Gen[T]=list[T] + +type = aliased +print(type(42)) + +# output + +type A = int +type Gen[T] = list[T] + +type = aliased +print(type(42)) diff --git a/tests/data/py_312/type_params.py b/tests/data/py_312/type_params.py new file mode 100644 index 00000000000..5f8ec43267c --- /dev/null +++ b/tests/data/py_312/type_params.py @@ -0,0 +1,57 @@ +def func [T ](): pass +async def func [ T ] (): pass +class C[ T ] : pass + +def all_in[T : int,U : (bytes, str),* Ts,**P](): pass + +def really_long[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine](): pass + +def even_longer[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine: WhatIfItHadABound](): pass + +def it_gets_worse[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine, ItCouldBeGenericOverMultipleTypeVars](): pass + +def magic[Trailing, Comma,](): pass + +# output + + +def func[T](): + pass + + +async def func[T](): + pass + + +class C[T]: + pass + + +def all_in[T: int, U: (bytes, str), *Ts, **P](): + pass + + +def really_long[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine +](): + pass + + +def even_longer[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine: WhatIfItHadABound +](): + pass + + +def it_gets_worse[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine, + ItCouldBeGenericOverMultipleTypeVars, +](): + pass + + +def magic[ + Trailing, + Comma, +](): + pass diff --git a/tests/data/simple_cases/comments2.py b/tests/data/simple_cases/comments2.py index 37e185abf4f..1487dc4b6e2 100644 --- a/tests/data/simple_cases/comments2.py +++ b/tests/data/simple_cases/comments2.py @@ -154,6 +154,9 @@ def _init_host(self, parsed) -> None: not parsed.hostname.strip()): pass + +a = "type comment with trailing space" # type: str + ####################### ### SECTION COMMENT ### ####################### @@ -332,6 +335,8 @@ def _init_host(self, parsed) -> None: pass +a = "type comment with trailing space" # type: str + ####################### ### SECTION COMMENT ### ####################### diff --git a/tests/data/simple_cases/fstring.py b/tests/data/simple_cases/fstring.py index 4b33231c01c..60560309376 100644 --- a/tests/data/simple_cases/fstring.py +++ b/tests/data/simple_cases/fstring.py @@ -7,6 +7,8 @@ f"\"{f'{nested} inner'}\" outer" f"space between opening braces: { {a for a in (1, 2, 3)}}" f'Hello \'{tricky + "example"}\'' +f"Tried directories {str(rootdirs)} \ +but none started with prefix {parentdir_prefix}" # output @@ -19,3 +21,5 @@ f"\"{f'{nested} inner'}\" outer" f"space between opening braces: { {a for a in (1, 2, 3)}}" f'Hello \'{tricky + "example"}\'' +f"Tried directories {str(rootdirs)} \ +but none started with prefix {parentdir_prefix}" diff --git a/tests/data/simple_cases/ignore_pyi.py b/tests/data/simple_cases/ignore_pyi.py new file mode 100644 index 00000000000..3ef61079bfe --- /dev/null +++ b/tests/data/simple_cases/ignore_pyi.py @@ -0,0 +1,41 @@ +def f(): # type: ignore + ... + +class x: # some comment + ... + +class y: + ... # comment + +# whitespace doesn't matter (note the next line has a trailing space and tab) +class z: + ... + +def g(): + # hi + ... + +def h(): + ... + # bye + +# output + +def f(): # type: ignore + ... + +class x: # some comment + ... + +class y: ... # comment + +# whitespace doesn't matter (note the next line has a trailing space and tab) +class z: ... + +def g(): + # hi + ... + +def h(): + ... + # bye diff --git a/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py b/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py new file mode 100644 index 00000000000..6ec8bb45408 --- /dev/null +++ b/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py @@ -0,0 +1,41 @@ +# This is a regression test. Issue #3737 + +a = ( # type: ignore + int( # type: ignore + int( # type: ignore + int( # type: ignore + 6 + ) + ) + ) +) + +b = ( + int( + 6 + ) +) + +print( "111") # type: ignore +print( "111" ) # type: ignore +print( "111" ) # type: ignore + + +# output + + +# This is a regression test. Issue #3737 + +a = ( # type: ignore + int( # type: ignore + int( # type: ignore + int(6) # type: ignore + ) + ) +) + +b = int(6) + +print("111") # type: ignore +print("111") # type: ignore +print("111") # type: ignore \ No newline at end of file diff --git a/tests/data/simple_cases/pep_604.py b/tests/data/simple_cases/pep_604.py new file mode 100644 index 00000000000..b68d59d6440 --- /dev/null +++ b/tests/data/simple_cases/pep_604.py @@ -0,0 +1,25 @@ +def some_very_long_name_function() -> my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | None: + pass + + +def some_very_long_name_function() -> my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | my_module.EvenMoreType | None: + pass + + +# output + + +def some_very_long_name_function() -> ( + my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | None +): + pass + + +def some_very_long_name_function() -> ( + my_module.Asdf + | my_module.AnotherType + | my_module.YetAnotherType + | my_module.EvenMoreType + | None +): + pass diff --git a/tests/test_black.py b/tests/test_black.py index 983e2ea36a2..7272412fa07 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -41,7 +41,7 @@ import pyink.files from pyink import Feature, TargetVersion from pyink import re_compile_maybe_verbose as compile_pattern -from pyink.cache import get_cache_dir, get_cache_file +from pyink.cache import FileData, get_cache_dir, get_cache_file from pyink.debug import DebugVisitor from pyink.output import color_diff, diff from pyink.report import Report @@ -104,6 +104,7 @@ class FakeContext(click.Context): def __init__(self) -> None: self.default_map: Dict[str, Any] = {} + self.params: Dict[str, Any] = {} # Dummy root, since most of the tests don't care about it self.obj: Dict[str, Any] = {"root": PROJECT_ROOT} @@ -148,8 +149,7 @@ def test_empty_ff(self) -> None: tmp_file = Path(pyink.dump_to_file()) try: self.assertFalse(ff(tmp_file, write_back=pyink.WriteBack.YES)) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -177,7 +177,7 @@ def test_one_empty_line_ff(self) -> None: ff(tmp_file, mode=mode, write_back=pyink.WriteBack.YES) ) with open(tmp_file, "rb") as f: - actual = f.read().decode("utf8") + actual = f.read().decode("utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -197,7 +197,7 @@ def test_piping(self) -> None: f"--line-length={pyink.DEFAULT_LINE_LENGTH}", f"--config={EMPTY_CONFIG}", ], - input=BytesIO(source.encode("utf8")), + input=BytesIO(source.encode("utf-8")), ) self.assertEqual(result.exit_code, 0) self.assertFormatEqual(expected, result.output) @@ -207,8 +207,8 @@ def test_piping(self) -> None: def test_piping_diff(self) -> None: diff_header = re.compile( - r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d " - r"\+\d\d\d\d" + r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d" + r"\+\d\d:\d\d" ) source, _ = read_data("simple_cases", "expression.py") expected, _ = read_data("simple_cases", "expression.diff") @@ -220,7 +220,7 @@ def test_piping_diff(self) -> None: f"--config={EMPTY_CONFIG}", ] result = BlackRunner().invoke( - pyink.main, args, input=BytesIO(source.encode("utf8")) + pyink.main, args, input=BytesIO(source.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) actual = diff_header.sub(DETERMINISTIC_HEADER, result.output) @@ -238,7 +238,7 @@ def test_piping_diff_with_color(self) -> None: f"--config={EMPTY_CONFIG}", ] result = BlackRunner().invoke( - pyink.main, args, input=BytesIO(source.encode("utf8")) + pyink.main, args, input=BytesIO(source.encode("utf-8")) ) actual = result.output # Again, the contents are checked in a different test, so only look for colors. @@ -271,13 +271,21 @@ def test_pep_572_version_detection(self) -> None: versions = pyink.detect_target_versions(root) self.assertIn(pyink.TargetVersion.PY38, versions) + def test_pep_695_version_detection(self) -> None: + for file in ("type_aliases", "type_params"): + source, _ = read_data("py_312", file) + root = pyink.lib2to3_parse(source) + features = pyink.get_features_used(root) + self.assertIn(pyink.Feature.TYPE_PARAMS, features) + versions = pyink.detect_target_versions(root) + self.assertIn(pyink.TargetVersion.PY312, versions) + def test_expression_ff(self) -> None: source, expected = read_data("simple_cases", "expression.py") tmp_file = Path(pyink.dump_to_file(source)) try: self.assertTrue(ff(tmp_file, write_back=pyink.WriteBack.YES)) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -291,7 +299,7 @@ def test_expression_diff(self) -> None: tmp_file = Path(pyink.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke( @@ -380,8 +388,7 @@ def test_skip_source_first_line(self) -> None: pyink.main, [str(tmp_file), "-x", f"--config={EMPTY_CONFIG}"] ) self.assertEqual(result.exit_code, 0) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") self.assertFormatEqual(source, actual) def test_skip_source_first_line_when_mixing_newlines(self) -> None: @@ -402,7 +409,7 @@ def test_skip_magic_trailing_comma(self) -> None: tmp_file = Path(pyink.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke( @@ -485,9 +492,7 @@ def test_false_positive_symlink_output_issue_3384(self) -> None: project_root = Path(THIS_DIR / "data" / "nested_gitignore_tests") working_directory = project_root / "root" target_abspath = working_directory / "child" - target_contents = ( - src.relative_to(working_directory) for src in target_abspath.iterdir() - ) + target_contents = list(target_abspath.iterdir()) def mock_n_calls(responses: List[bool]) -> Callable[[], bool]: def _mocked_calls() -> bool: @@ -500,11 +505,11 @@ def _mocked_calls() -> bool: with patch("pathlib.Path.iterdir", return_value=target_contents), patch( "pathlib.Path.cwd", return_value=working_directory ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): - ctx = FakeContext() - ctx.obj["root"] = project_root + # Note that the root folder (project_root) isn't the folder + # named "root" (aka working_directory) report = MagicMock(verbose=True) pyink.get_sources( - ctx=ctx, + root=project_root, src=("./child",), quiet=False, verbose=True, @@ -520,7 +525,7 @@ def _mocked_calls() -> bool: for _, mock_args, _ in report.path_ignored.mock_calls ), "A symbolic link was reported." report.path_ignored.assert_called_once_with( - Path("child", "b.py"), "matches a .gitignore file content" + Path("root", "child", "b.py"), "matches a .gitignore file content" ) def test_report_verbose(self) -> None: @@ -1071,7 +1076,7 @@ def test_works_in_mono_process_only_environment(self) -> None: (workspace / "one.py").resolve(), (workspace / "two.py").resolve(), ]: - f.write_text('print("hello")\n') + f.write_text('print("hello")\n', encoding="utf-8") self.invokeBlack([str(workspace)]) @event_loop() @@ -1108,16 +1113,14 @@ def test_single_file_force_pyi(self) -> None: contents, expected = read_data("miscellaneous", "force_pyi") with cache_dir() as workspace: path = (workspace / "file.py").resolve() - with open(path, "w") as fh: - fh.write(contents) + path.write_text(contents, encoding="utf-8") self.invokeBlack([str(path), "--pyi"]) - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") # verify cache with --pyi is separate - pyi_cache = pyink.read_cache(pyi_mode) - self.assertIn(str(path), pyi_cache) - normal_cache = pyink.read_cache(DEFAULT_MODE) - self.assertNotIn(str(path), normal_cache) + pyi_cache = pyink.Cache.read(pyi_mode) + assert not pyi_cache.is_changed(path) + normal_cache = pyink.Cache.read(DEFAULT_MODE) + assert normal_cache.is_changed(path) self.assertFormatEqual(expected, actual) pyink.assert_equivalent(contents, actual) pyink.assert_stable(contents, actual, pyi_mode) @@ -1133,24 +1136,22 @@ def test_multi_file_force_pyi(self) -> None: (workspace / "file2.py").resolve(), ] for path in paths: - with open(path, "w") as fh: - fh.write(contents) + path.write_text(contents, encoding="utf-8") self.invokeBlack([str(p) for p in paths] + ["--pyi"]) for path in paths: - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --pyi is separate - pyi_cache = pyink.read_cache(pyi_mode) - normal_cache = pyink.read_cache(reg_mode) + pyi_cache = pyink.Cache.read(pyi_mode) + normal_cache = pyink.Cache.read(reg_mode) for path in paths: - self.assertIn(str(path), pyi_cache) - self.assertNotIn(str(path), normal_cache) + assert not pyi_cache.is_changed(path) + assert normal_cache.is_changed(path) def test_pipe_force_pyi(self) -> None: source, expected = read_data("miscellaneous", "force_pyi") result = CliRunner().invoke( - pyink.main, ["-", "-q", "--pyi"], input=BytesIO(source.encode("utf8")) + pyink.main, ["-", "-q", "--pyi"], input=BytesIO(source.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) actual = result.output @@ -1162,16 +1163,14 @@ def test_single_file_force_py36(self) -> None: source, expected = read_data("miscellaneous", "force_py36") with cache_dir() as workspace: path = (workspace / "file.py").resolve() - with open(path, "w") as fh: - fh.write(source) + path.write_text(source, encoding="utf-8") self.invokeBlack([str(path), *PY36_ARGS]) - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") # verify cache with --target-version is separate - py36_cache = pyink.read_cache(py36_mode) - self.assertIn(str(path), py36_cache) - normal_cache = pyink.read_cache(reg_mode) - self.assertNotIn(str(path), normal_cache) + py36_cache = pyink.Cache.read(py36_mode) + assert not py36_cache.is_changed(path) + normal_cache = pyink.Cache.read(reg_mode) + assert normal_cache.is_changed(path) self.assertEqual(actual, expected) @event_loop() @@ -1185,26 +1184,24 @@ def test_multi_file_force_py36(self) -> None: (workspace / "file2.py").resolve(), ] for path in paths: - with open(path, "w") as fh: - fh.write(source) + path.write_text(source, encoding="utf-8") self.invokeBlack([str(p) for p in paths] + PY36_ARGS) for path in paths: - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --target-version is separate - pyi_cache = pyink.read_cache(py36_mode) - normal_cache = pyink.read_cache(reg_mode) + pyi_cache = pyink.Cache.read(py36_mode) + normal_cache = pyink.Cache.read(reg_mode) for path in paths: - self.assertIn(str(path), pyi_cache) - self.assertNotIn(str(path), normal_cache) + assert not pyi_cache.is_changed(path) + assert normal_cache.is_changed(path) def test_pipe_force_py36(self) -> None: source, expected = read_data("miscellaneous", "force_py36") result = CliRunner().invoke( pyink.main, ["-", "-q", "--target-version=py36"], - input=BytesIO(source.encode("utf8")), + input=BytesIO(source.encode("utf-8")), ) self.assertEqual(result.exit_code, 0) actual = result.output @@ -1457,11 +1454,11 @@ def test_preserves_line_endings_via_stdin(self) -> None: contents = nl.join(["def f( ):", " pass"]) runner = BlackRunner() result = runner.invoke( - pyink.main, ["-", "--fast"], input=BytesIO(contents.encode("utf8")) + pyink.main, ["-", "--fast"], input=BytesIO(contents.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) output = result.stdout_bytes - self.assertIn(nl.encode("utf8"), output) + self.assertIn(nl.encode("utf-8"), output) if nl == "\n": self.assertNotIn(b"\r\n", output) @@ -1480,30 +1477,6 @@ def test_assert_equivalent_different_asts(self) -> None: with self.assertRaises(AssertionError): pyink.assert_equivalent("{}", "None") - def test_shhh_click(self) -> None: - try: - from click import _unicodefun # type: ignore - except ImportError: - self.skipTest("Incompatible Click version") - - if not hasattr(_unicodefun, "_verify_python_env"): - self.skipTest("Incompatible Click version") - - # First, let's see if Click is crashing with a preferred ASCII charset. - with patch("locale.getpreferredencoding") as gpe: - gpe.return_value = "ASCII" - with self.assertRaises(RuntimeError): - _unicodefun._verify_python_env() - # Now, let's silence Click... - pyink.patch_click() - # ...and confirm it's silent. - with patch("locale.getpreferredencoding") as gpe: - gpe.return_value = "ASCII" - try: - _unicodefun._verify_python_env() - except RuntimeError as re: - self.fail(f"`patch_click()` failed, exception still raised: {re}") - def test_root_logger_not_used_directly(self) -> None: def fail(*args: Any, **kwargs: Any) -> None: self.fail("Record created with root logger") @@ -1557,14 +1530,25 @@ def test_infer_target_version(self) -> None: for version, expected in [ ("3.6", [TargetVersion.PY36]), ("3.11.0rc1", [TargetVersion.PY311]), - (">=3.10", [TargetVersion.PY310, TargetVersion.PY311]), - (">=3.10.6", [TargetVersion.PY310, TargetVersion.PY311]), + (">=3.10", [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312]), + ( + ">=3.10.6", + [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + ), ("<3.6", [TargetVersion.PY33, TargetVersion.PY34, TargetVersion.PY35]), (">3.7,<3.10", [TargetVersion.PY38, TargetVersion.PY39]), - (">3.7,!=3.8,!=3.9", [TargetVersion.PY310, TargetVersion.PY311]), + ( + ">3.7,!=3.8,!=3.9", + [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + ), ( "> 3.9.4, != 3.10.3", - [TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311], + [ + TargetVersion.PY39, + TargetVersion.PY310, + TargetVersion.PY311, + TargetVersion.PY312, + ], ), ( "!=3.3,!=3.4", @@ -1576,6 +1560,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311, + TargetVersion.PY312, ], ), ( @@ -1590,6 +1575,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311, + TargetVersion.PY312, ], ), ("==3.8.*", [TargetVersion.PY38]), @@ -1622,6 +1608,39 @@ def test_read_pyproject_toml(self) -> None: self.assertEqual(config["exclude"], r"\.pyi?$") self.assertEqual(config["include"], r"\.py?$") + def test_read_pyproject_toml_from_stdin(self) -> None: + with TemporaryDirectory() as workspace: + root = Path(workspace) + + src_dir = root / "src" + src_dir.mkdir() + + src_pyproject = src_dir / "pyproject.toml" + src_pyproject.touch() + + test_toml_content = (THIS_DIR / "test.toml").read_text(encoding="utf-8") + src_pyproject.write_text(test_toml_content, encoding="utf-8") + + src_python = src_dir / "foo.py" + src_python.touch() + + fake_ctx = FakeContext() + fake_ctx.params["src"] = ("-",) + fake_ctx.params["stdin_filename"] = str(src_python) + + with change_directory(root): + pyink.read_pyproject_toml(fake_ctx, FakeParameter(), None) + + config = fake_ctx.default_map + self.assertEqual(config["verbose"], "1") + self.assertEqual(config["check"], "no") + self.assertEqual(config["diff"], "y") + self.assertEqual(config["color"], "True") + self.assertEqual(config["line_length"], "79") + self.assertEqual(config["target_version"], ["py36", "py37", "py38"]) + self.assertEqual(config["exclude"], r"\.pyi?$") + self.assertEqual(config["include"], r"\.py?$") + @pytest.mark.incompatible_with_mypyc def test_find_project_root(self) -> None: with TemporaryDirectory() as workspace: @@ -1752,7 +1771,7 @@ def test_bpo_2142_workaround(self) -> None: tmp_file = Path(pyink.dump_to_file(source, ensure_final_newline=False)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke(pyink.main, ["--diff", str(tmp_file)]) @@ -1953,22 +1972,23 @@ def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: cache_file = get_cache_file(mode) - cache_file.write_text("this is not a pickle") - assert pyink.read_cache(mode) == {} + cache_file.write_text("this is not a pickle", encoding="utf-8") + assert pyink.Cache.read(mode).file_data == {} src = (workspace / "test.py").resolve() - src.write_text("print('hello')") + src.write_text("print('hello')", encoding="utf-8") invokeBlack([str(src)]) - cache = pyink.read_cache(mode) - assert str(src) in cache + cache = pyink.Cache.read(mode) + assert not cache.is_changed(src) def test_cache_single_file_already_cached(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() - src.write_text("print('hello')") - pyink.write_cache({}, [src], mode) + src.write_text("print('hello')", encoding="utf-8") + cache = pyink.Cache.read(mode) + cache.write([src]) invokeBlack([str(src)]) - assert src.read_text() == "print('hello')" + assert src.read_text(encoding="utf-8") == "print('hello')" @event_loop() def test_cache_multiple_files(self) -> None: @@ -1977,30 +1997,27 @@ def test_cache_multiple_files(self) -> None: "concurrent.futures.ProcessPoolExecutor", new=ThreadPoolExecutor ): one = (workspace / "one.py").resolve() - with one.open("w") as fobj: - fobj.write("print('hello')") + one.write_text("print('hello')", encoding="utf-8") two = (workspace / "two.py").resolve() - with two.open("w") as fobj: - fobj.write("print('hello')") - pyink.write_cache({}, [one], mode) + two.write_text("print('hello')", encoding="utf-8") + cache = pyink.Cache.read(mode) + cache.write([one]) invokeBlack([str(workspace)]) - with one.open("r") as fobj: - assert fobj.read() == "print('hello')" - with two.open("r") as fobj: - assert fobj.read() == 'print("hello")\n' - cache = pyink.read_cache(mode) - assert str(one) in cache - assert str(two) in cache + assert one.read_text(encoding="utf-8") == "print('hello')" + assert two.read_text(encoding="utf-8") == 'print("hello")\n' + cache = pyink.Cache.read(mode) + assert not cache.is_changed(one) + assert not cache.is_changed(two) + @pytest.mark.incompatible_with_mypyc @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) def test_no_cache_when_writeback_diff(self, color: bool) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() - with src.open("w") as fobj: - fobj.write("print('hello')") - with patch("pyink.read_cache") as read_cache, patch( - "pyink.write_cache" + src.write_text("print('hello')", encoding="utf-8") + with patch.object(pyink.Cache, "read") as read_cache, patch.object( + pyink.Cache, "write" ) as write_cache: cmd = [str(src), "--diff"] if color: @@ -2008,8 +2025,8 @@ def test_no_cache_when_writeback_diff(self, color: bool) -> None: invokeBlack(cmd) cache_file = get_cache_file(mode) assert cache_file.exists() is False + read_cache.assert_called_once() write_cache.assert_not_called() - read_cache.assert_not_called() @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) @event_loop() @@ -2017,8 +2034,7 @@ def test_output_locking_when_writeback_diff(self, color: bool) -> None: with cache_dir() as workspace: for tag in range(0, 4): src = (workspace / f"test{tag}.py").resolve() - with src.open("w") as fobj: - fobj.write("print('hello')") + src.write_text("print('hello')", encoding="utf-8") with patch( "pyink.concurrency.Manager", wraps=multiprocessing.Manager ) as mgr: @@ -2043,18 +2059,19 @@ def test_no_cache_when_stdin(self) -> None: def test_read_cache_no_cachefile(self) -> None: mode = DEFAULT_MODE with cache_dir(): - assert pyink.read_cache(mode) == {} + assert pyink.Cache.read(mode).file_data == {} def test_write_cache_read_cache(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() src.touch() - pyink.write_cache({}, [src], mode) - cache = pyink.read_cache(mode) - assert str(src) in cache - assert cache[str(src)] == pyink.get_cache_info(src) + write_cache = pyink.Cache.read(mode) + write_cache.write([src]) + read_cache = pyink.Cache.read(mode) + assert not read_cache.is_changed(src) + @pytest.mark.incompatible_with_mypyc def test_filter_cached(self) -> None: with TemporaryDirectory() as workspace: path = Path(workspace) @@ -2064,21 +2081,67 @@ def test_filter_cached(self) -> None: uncached.touch() cached.touch() cached_but_changed.touch() - cache = { - str(cached): pyink.get_cache_info(cached), - str(cached_but_changed): (0.0, 0), - } - todo, done = pyink.cache.filter_cached( - cache, {uncached, cached, cached_but_changed} - ) + cache = pyink.Cache.read(DEFAULT_MODE) + + orig_func = pyink.Cache.get_file_data + + def wrapped_func(path: Path) -> FileData: + if path == cached: + return orig_func(path) + if path == cached_but_changed: + return FileData(0.0, 0, "") + raise AssertionError + + with patch.object(pyink.Cache, "get_file_data", side_effect=wrapped_func): + cache.write([cached, cached_but_changed]) + todo, done = cache.filtered_cached({uncached, cached, cached_but_changed}) assert todo == {uncached, cached_but_changed} assert done == {cached} + def test_filter_cached_hash(self) -> None: + with TemporaryDirectory() as workspace: + path = Path(workspace) + src = (path / "test.py").resolve() + src.write_text("print('hello')", encoding="utf-8") + st = src.stat() + cache = pyink.Cache.read(DEFAULT_MODE) + cache.write([src]) + cached_file_data = cache.file_data[str(src)] + + todo, done = cache.filtered_cached([src]) + assert todo == set() + assert done == {src} + assert cached_file_data.st_mtime == st.st_mtime + + # Modify st_mtime + cached_file_data = cache.file_data[str(src)] = FileData( + cached_file_data.st_mtime - 1, + cached_file_data.st_size, + cached_file_data.hash, + ) + todo, done = cache.filtered_cached([src]) + assert todo == set() + assert done == {src} + assert cached_file_data.st_mtime < st.st_mtime + assert cached_file_data.st_size == st.st_size + assert cached_file_data.hash == pyink.Cache.hash_digest(src) + + # Modify contents + src.write_text("print('hello world')", encoding="utf-8") + new_st = src.stat() + todo, done = cache.filtered_cached([src]) + assert todo == {src} + assert done == set() + assert cached_file_data.st_mtime < new_st.st_mtime + assert cached_file_data.st_size != new_st.st_size + assert cached_file_data.hash != pyink.Cache.hash_digest(src) + def test_write_cache_creates_directory_if_needed(self) -> None: mode = DEFAULT_MODE with cache_dir(exists=False) as workspace: assert not workspace.exists() - pyink.write_cache({}, [], mode) + cache = pyink.Cache.read(mode) + cache.write([]) assert workspace.exists() @event_loop() @@ -2088,21 +2151,21 @@ def test_failed_formatting_does_not_get_cached(self) -> None: "concurrent.futures.ProcessPoolExecutor", new=ThreadPoolExecutor ): failing = (workspace / "failing.py").resolve() - with failing.open("w") as fobj: - fobj.write("not actually python") + failing.write_text("not actually python", encoding="utf-8") clean = (workspace / "clean.py").resolve() - with clean.open("w") as fobj: - fobj.write('print("hello")\n') + clean.write_text('print("hello")\n', encoding="utf-8") invokeBlack([str(workspace)], exit_code=123) - cache = pyink.read_cache(mode) - assert str(failing) not in cache - assert str(clean) in cache + cache = pyink.Cache.read(mode) + assert cache.is_changed(failing) + assert not cache.is_changed(clean) def test_write_cache_write_fail(self) -> None: mode = DEFAULT_MODE - with cache_dir(), patch.object(Path, "open") as mock: - mock.side_effect = OSError - pyink.write_cache({}, [], mode) + with cache_dir(): + cache = pyink.Cache.read(mode) + with patch.object(Path, "open") as mock: + mock.side_effect = OSError + cache.write([]) def test_read_cache_line_lengths(self) -> None: mode = DEFAULT_MODE @@ -2110,18 +2173,19 @@ def test_read_cache_line_lengths(self) -> None: with cache_dir() as workspace: path = (workspace / "file.py").resolve() path.touch() - pyink.write_cache({}, [path], mode) - one = pyink.read_cache(mode) - assert str(path) in one - two = pyink.read_cache(short_mode) - assert str(path) not in two + cache = pyink.Cache.read(mode) + cache.write([path]) + one = pyink.Cache.read(mode) + assert not one.is_changed(path) + two = pyink.Cache.read(short_mode) + assert two.is_changed(path) def assert_collected_sources( src: Sequence[Union[str, Path]], expected: Sequence[Union[str, Path]], *, - ctx: Optional[FakeContext] = None, + root: Optional[Path] = None, exclude: Optional[str] = None, include: Optional[str] = None, extend_exclude: Optional[str] = None, @@ -2137,7 +2201,7 @@ def assert_collected_sources( ) gs_force_exclude = None if force_exclude is None else compile_pattern(force_exclude) collected = pyink.get_sources( - ctx=ctx or FakeContext(), + root=root or THIS_DIR, src=gs_src, quiet=False, verbose=False, @@ -2173,9 +2237,7 @@ def test_gitignore_used_as_default(self) -> None: base / "b/.definitely_exclude/a.pyi", ] src = [base / "b/"] - ctx = FakeContext() - ctx.obj["root"] = base - assert_collected_sources(src, expected, ctx=ctx, extend_exclude=r"/exclude/") + assert_collected_sources(src, expected, root=base, extend_exclude=r"/exclude/") def test_gitignore_used_on_multiple_sources(self) -> None: root = Path(DATA_DIR / "gitignore_used_on_multiple_sources") @@ -2183,10 +2245,8 @@ def test_gitignore_used_on_multiple_sources(self) -> None: root / "dir1" / "b.py", root / "dir2" / "b.py", ] - ctx = FakeContext() - ctx.obj["root"] = root src = [root / "dir1", root / "dir2"] - assert_collected_sources(src, expected, ctx=ctx) + assert_collected_sources(src, expected, root=root) @patch("pyink.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_exclude_for_issue_1572(self) -> None: @@ -2292,9 +2352,7 @@ def test_gitignore_that_ignores_subfolders(self) -> None: # If gitignore with */* is in root root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests" / "subdir") expected = [root / "b.py"] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([root], expected, ctx=ctx) + assert_collected_sources([root], expected, root=root) # If .gitignore with */* is nested root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests") @@ -2302,17 +2360,13 @@ def test_gitignore_that_ignores_subfolders(self) -> None: root / "a.py", root / "subdir" / "b.py", ] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([root], expected, ctx=ctx) + assert_collected_sources([root], expected, root=root) # If command is executed from outer dir root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests") target = root / "subdir" expected = [target / "b.py"] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([target], expected, ctx=ctx) + assert_collected_sources([target], expected, root=root) def test_empty_include(self) -> None: path = DATA_DIR / "include_exclude_tests" @@ -2345,38 +2399,48 @@ def test_extend_exclude(self) -> None: ) @pytest.mark.incompatible_with_mypyc - def test_symlink_out_of_root_directory(self) -> None: + def test_symlinks(self) -> None: path = MagicMock() root = THIS_DIR.resolve() - child = MagicMock() include = re.compile(pyink.DEFAULT_INCLUDES) exclude = re.compile(pyink.DEFAULT_EXCLUDES) report = pyink.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) - # `child` should behave like a symlink which resolved path is clearly - # outside of the `root` directory. - path.iterdir.return_value = [child] - child.resolve.return_value = Path("/a/b/c") - child.as_posix.return_value = "/a/b/c" - try: - list( - pyink.gen_python_files( - path.iterdir(), - root, - include, - exclude, - None, - None, - report, - {path: gitignore}, - verbose=False, - quiet=False, - ) + + regular = MagicMock() + outside_root_symlink = MagicMock() + ignored_symlink = MagicMock() + + path.iterdir.return_value = [regular, outside_root_symlink, ignored_symlink] + + regular.absolute.return_value = root / "regular.py" + regular.resolve.return_value = root / "regular.py" + regular.is_dir.return_value = False + + outside_root_symlink.absolute.return_value = root / "symlink.py" + outside_root_symlink.resolve.return_value = Path("/nowhere") + + ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + + files = list( + pyink.gen_python_files( + path.iterdir(), + root, + include, + exclude, + None, + None, + report, + {path: gitignore}, + verbose=False, + quiet=False, ) - except ValueError as ve: - pytest.fail(f"`get_python_files_in_dir()` failed: {ve}") + ) + assert files == [regular] + path.iterdir.assert_called_once() - child.resolve.assert_called_once() + outside_root_symlink.resolve.assert_called_once() + ignored_symlink.resolve.assert_not_called() @patch("pyink.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: @@ -2555,6 +2619,41 @@ def test_pyink_use_majority_quotes(self) -> None: assert diff in self.decode_and_normalized(result.stdout_bytes) +class TestDeFactoAPI: + """Test that certain symbols that are commonly used externally keep working. + + We don't (yet) formally expose an API (see issue #779), but we should endeavor to + keep certain functions that external users commonly rely on working. + + """ + + def test_format_str(self) -> None: + # format_str and Mode should keep working + assert ( + pyink.format_str("print('hello')", mode=pyink.Mode()) == 'print("hello")\n' + ) + + # you can pass line length + assert ( + pyink.format_str("print('hello')", mode=pyink.Mode(line_length=42)) + == 'print("hello")\n' + ) + + # invalid input raises InvalidInput + with pytest.raises(pyink.InvalidInput): + pyink.format_str("syntax error", mode=pyink.Mode()) + + def test_format_file_contents(self) -> None: + # You probably should be using format_str() instead, but let's keep + # this one around since people do use it + assert ( + pyink.format_file_contents("x=1", fast=True, mode=pyink.Mode()) == "x = 1\n" + ) + + with pytest.raises(pyink.NothingChanged): + pyink.format_file_contents("x = 1\n", fast=True, mode=pyink.Mode()) + + try: with open(pyink.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() diff --git a/tests/test_format.py b/tests/test_format.py index 3637124d340..8f41183281a 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -33,9 +33,10 @@ def check_file( @pytest.mark.parametrize("filename", all_data_cases("simple_cases")) def test_simple_format(filename: str) -> None: magic_trailing_comma = filename != "skip_magic_trailing_comma" - check_file( - "simple_cases", filename, pyink.Mode(magic_trailing_comma=magic_trailing_comma) + mode = pyink.Mode( + magic_trailing_comma=magic_trailing_comma, is_pyi=filename.endswith("_pyi") ) + check_file("simple_cases", filename, mode) @pytest.mark.parametrize("filename", all_data_cases("preview")) @@ -64,6 +65,13 @@ def test_preview_context_managers_targeting_py39() -> None: assert_format(source, expected, mode, minimum_version=(3, 9)) +@pytest.mark.parametrize("filename", all_data_cases("preview_py_310")) +def test_preview_python_310(filename: str) -> None: + source, expected = read_data("preview_py_310", filename) + mode = pyink.Mode(target_versions={pyink.TargetVersion.PY310}, preview=True) + assert_format(source, expected, mode, minimum_version=(3, 10)) + + @pytest.mark.parametrize( "filename", all_data_cases("preview_context_managers/auto_detect") ) @@ -143,6 +151,13 @@ def test_python_311(filename: str) -> None: assert_format(source, expected, mode, minimum_version=(3, 11)) +@pytest.mark.parametrize("filename", all_data_cases("py_312")) +def test_python_312(filename: str) -> None: + source, expected = read_data("py_312", filename) + mode = pyink.Mode(target_versions={pyink.TargetVersion.PY312}) + assert_format(source, expected, mode, minimum_version=(3, 12)) + + @pytest.mark.parametrize("filename", all_data_cases("fast")) def test_fast_cases(filename: str) -> None: source, expected = read_data("fast", filename) @@ -195,9 +210,9 @@ def test_stub() -> None: assert_format(source, expected, mode) -def test_nested_class_stub() -> None: +def test_nested_stub() -> None: mode = replace(DEFAULT_MODE, is_pyi=True, preview=True) - source, expected = read_data("miscellaneous", "nested_class_stub.pyi") + source, expected = read_data("miscellaneous", "nested_stub.pyi") assert_format(source, expected, mode) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index b736ede6adb..b7bdc189ba9 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -77,7 +77,7 @@ def test_trailing_semicolon_noop() -> None: [ pytest.param(JUPYTER_MODE, id="default mode"), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), id="custom cell magics mode", ), ], @@ -100,7 +100,7 @@ def test_cell_magic_noop() -> None: [ pytest.param(JUPYTER_MODE, id="default mode"), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), id="custom cell magics mode", ), ], @@ -183,7 +183,7 @@ def test_cell_magic_with_magic() -> None: id="No change when cell magic not registered", ), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), "%%custom_python_magic -n1 -n2\nx=2", pytest.raises(NothingChanged), id="No change when other cell magics registered", @@ -439,19 +439,14 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( jupyter_dependencies_are_installed.cache_clear() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) - monkeypatch.setattr( - "pyink.jupyter_dependencies_are_installed", lambda verbose, quiet: False - ) + tmp_nb.write_bytes(nb.read_bytes()) + monkeypatch.setattr("pyink.jupyter_dependencies_are_installed", lambda warn: False) result = runner.invoke( main, [str(tmp_path / "notebook.ipynb"), f"--config={EMPTY_CONFIG}"] ) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() - monkeypatch.setattr( - "pyink.jupyter_dependencies_are_installed", lambda verbose, quiet: True - ) + monkeypatch.setattr("pyink.jupyter_dependencies_are_installed", lambda warn: True) result = runner.invoke( main, [str(tmp_path / "notebook.ipynb"), f"--config={EMPTY_CONFIG}"] ) @@ -465,16 +460,15 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( jupyter_dependencies_are_installed.cache_clear() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) monkeypatch.setattr( - "pyink.files.jupyter_dependencies_are_installed", lambda verbose, quiet: False + "pyink.files.jupyter_dependencies_are_installed", lambda warn: False ) result = runner.invoke(main, [str(tmp_path), f"--config={EMPTY_CONFIG}"]) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() monkeypatch.setattr( - "pyink.files.jupyter_dependencies_are_installed", lambda verbose, quiet: True + "pyink.files.jupyter_dependencies_are_installed", lambda warn: True ) result = runner.invoke(main, [str(tmp_path), f"--config={EMPTY_CONFIG}"]) assert "reformatted" in result.output @@ -483,8 +477,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( def test_ipynb_flag(tmp_path: pathlib.Path) -> None: nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.a_file_extension_which_is_definitely_not_ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) result = runner.invoke( main, [ diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index abdbfc1d234..65522f5581e 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -27,8 +27,7 @@ def test_ipynb_diff_with_no_change_dir(tmp_path: pathlib.Path) -> None: runner = CliRunner() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) result = runner.invoke(main, [str(tmp_path)]) expected_output = ( "Skipping .ipynb files as Jupyter dependencies are not installed.\n" diff --git a/tests/test_trans.py b/tests/test_trans.py index 927f02c6cfb..d5d21b176a5 100644 --- a/tests/test_trans.py +++ b/tests/test_trans.py @@ -13,7 +13,7 @@ def check( # a glance than only spans assert len(spans) == len(expected_slices) for (i, j), slice in zip(spans, expected_slices): - assert len(string[i:j]) == j - i + assert 0 <= i <= j <= len(string) assert string[i:j] == slice assert spans == expected_spans diff --git a/tox.ini b/tox.ini index f19fd927de1..ff039f075ce 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,9 @@ isolated_build = true envlist = {,ci-}py{37,38,39,310,311,py3},fuzz,run_self [testenv] -setenv = PYTHONPATH = {toxinidir}/src +setenv = + PYTHONPATH = {toxinidir}/src + PYTHONWARNDEFAULTENCODING = 1 skip_install = True # We use `recreate=True` because otherwise, on the second run of `tox -e py`, # the `no_jupyter` tests would run with the jupyter extra dependencies installed. @@ -37,19 +39,15 @@ deps = ; remove this when pypy releases the bugfix commands = pip install -e .[d] - coverage erase pytest tests \ --run-optional no_jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ - --cov {posargs} + ci: --numprocesses 1 pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ - --cov --cov-append {posargs} - coverage report + ci: --numprocesses 1 [testenv:{,ci-}311] setenv =