diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a3106d04aae..04e30e727bd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -44,7 +44,9 @@ jobs: tags: pyfound/black:latest,pyfound/black:${{ env.GIT_TAG }} - name: Build and push latest_release tag - if: ${{ github.event_name == 'release' && github.event.action == 'published' }} + if: + ${{ github.event_name == 'release' && github.event.action == 'published' && + !github.event.release.prerelease }} uses: docker/build-push-action@v3 with: context: . @@ -52,5 +54,16 @@ jobs: push: true tags: pyfound/black:latest_release + - name: Build and push latest_prerelease tag + if: + ${{ github.event_name == 'release' && github.event.action == 'published' && + github.event.release.prerelease }} + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: pyfound/black:latest_prerelease + - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index ebb8a9fda9e..373e1500ee9 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-311.yml b/.github/workflows/test-311.yml deleted file mode 100644 index c2da2465ad5..00000000000 --- a/.github/workflows/test-311.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Test 3.11 without aiohttp extensions - -on: - push: - paths-ignore: - - "docs/**" - - "*.md" - - pull_request: - paths-ignore: - - "docs/**" - - "*.md" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - -jobs: - main: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. Without this if check, checks are duplicated since - # internal PRs match both the push and pull_request events. - if: - github.event_name == 'push' || github.event.pull_request.head.repo.full_name != - github.repository - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.11.0-rc - 3.11"] - os: [ubuntu-latest, macOS-latest, windows-latest] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install tox - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade tox - - - name: Run tests via tox - run: | - python -m tox -e 311 - - - name: Format ourselves - run: | - python -m pip install . - python -m black --check src/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 372d1fd5d38..3ca2a469147 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "pypy-3.7", "pypy-3.8"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7", "pypy-3.8"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: diff --git a/CHANGES.md b/CHANGES.md index 1b946b2d50f..5c277c4d7d5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,16 @@ +- Improve the performance on large expressions that contain many strings (#3467) - Fix a crash in preview style with assert + parenthesized string (#3415) +- Fix crashes in preview style with walrus operators used in function return annotations + and except clauses (#3423) +- Fix a crash in preview advanced string processing where mixed implicitly concatenated + regular and f-strings start with an empty span (#3463) +- Fix a crash in preview advanced string processing where a standalone comment is placed + before a dict's value (#3469) +- Fix an issue where extra empty lines are added when a decorator has `# fmt: skip` + applied or there is a standalone comment between decorators (#3470) - Do not put the closing quotes in a docstring on a separate line, even if the line is too long (#3430) - Long values in dict literals are now wrapped in parentheses; correspondingly @@ -24,6 +33,8 @@ string lambda values are now wrapped in parentheses (#3440) - Let string splitters respect [East Asian Width](https://www.unicode.org/reports/tr11/) (#3445) +- Exclude string type annotations from improved string processing; fix crash when the + return type annotation is stringified and spans across multiple lines (#3462) ### Configuration @@ -35,6 +46,9 @@ - Upgrade mypyc from `0.971` to `0.991` so mypycified _Black_ can be built on armv7 (#3380) +- Drop specific support for the `tomli` requirement on 3.11 alpha releases, working + around a bug that would cause the requirement not to be installed on any non-final + Python releases (#3448) ### Parser @@ -59,11 +73,17 @@ +- Move 3.11 CI to normal flow now all dependencies support 3.11 (#3446) +- Docker: Add new `latest_prerelease` tag automation to follow latest black alpha + release on docker images (#3465) + ### Documentation +- Expand `vim-plug` installation instructions to offer more explicit options (#3468) + ## 22.12.0 ### Preview style diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 0778c6a72f1..a8b7978c4d7 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -111,16 +111,51 @@ Configuration: - `g:black_fast` (defaults to `0`) - `g:black_linelength` (defaults to `88`) - `g:black_skip_string_normalization` (defaults to `0`) +- `g:black_skip_magic_trailing_comma` (defaults to `0`) - `g:black_virtualenv` (defaults to `~/.vim/black` or `~/.local/share/nvim/black`) +- `g:black_use_virtualenv` (defaults to `1`) +- `g:black_target_version` (defaults to `""`) - `g:black_quiet` (defaults to `0`) - `g:black_preview` (defaults to `0`) +#### Installation + +This plugin **requires Vim 7.0+ built with Python 3.7+ support**. It needs Python 3.7 to +be able to run _Black_ inside the Vim process which is much faster than calling an +external command. + +##### `vim-plug` + To install with [vim-plug](https://github.com/junegunn/vim-plug): +_Black_'s `stable` branch tracks official version updates, and can be used to simply +follow the most recent stable version. + ``` Plug 'psf/black', { 'branch': 'stable' } ``` +Another option which is a bit more explicit and offers more control is to use +`vim-plug`'s `tag` option with a shell wildcard. This will resolve to the latest tag +which matches the given pattern. + +The following matches all stable versions (see the +[Release Process](../contributing/release_process.md) section for documentation of +version scheme used by Black): + +``` +Plug 'psf/black', { 'tag': '*.*.*' } +``` + +and the following demonstrates pinning to a specific year's stable style (2022 in this +case): + +``` +Plug 'psf/black', { 'tag': '22.*.*' } +``` + +##### Vundle + or with [Vundle](https://github.com/VundleVim/Vundle.vim): ``` @@ -134,6 +169,14 @@ $ cd ~/.vim/bundle/black $ git checkout origin/stable -b stable ``` +##### Arch Linux + +On Arch Linux, the plugin is shipped with the +[`python-black`](https://archlinux.org/packages/community/any/python-black/) package, so +you can start using it in Vim after install with no additional setup. + +##### Vim 8 Native Plugin Management + or you can copy the plugin files from [plugin/black.vim](https://github.com/psf/black/blob/stable/plugin/black.vim) and [autoload/black.vim](https://github.com/psf/black/blob/stable/autoload/black.vim). @@ -148,9 +191,7 @@ curl https://raw.githubusercontent.com/psf/black/stable/autoload/black.vim -o ~/ Let me know if this requires any changes to work with Vim 8's builtin `packadd`, or Pathogen, and so on. -This plugin **requires Vim 7.0+ built with Python 3.7+ support**. It needs Python 3.7 to -be able to run _Black_ inside the Vim process which is much faster than calling an -external command. +#### Usage On first run, the plugin creates its own virtualenv using the right Python version and automatically installs _Black_. You can upgrade it later by calling `:BlackUpgrade` and @@ -187,6 +228,8 @@ To run _Black_ on a key press (e.g. F9 below), add this: nnoremap :Black ``` +#### Troubleshooting + **How to get Vim with Python 3.6?** On Ubuntu 17.10 Vim comes with Python 3.6 by default. On macOS with Homebrew run: `brew install vim`. When building Vim from source, use: `./configure --enable-python3interp=yes`. There's many guides online how to do diff --git a/docs/usage_and_configuration/black_docker_image.md b/docs/usage_and_configuration/black_docker_image.md index 8de566ea270..85aec91ef1c 100644 --- a/docs/usage_and_configuration/black_docker_image.md +++ b/docs/usage_and_configuration/black_docker_image.md @@ -10,6 +10,11 @@ _Black_ images with the following tags are available: - `latest_release` - tag created when a new version of _Black_ is released.\ ℹ Recommended for users who want to use released versions of _Black_. It maps to [the latest release](https://github.com/psf/black/releases/latest) of _Black_. +- `latest_prerelease` - tag created when a new alpha (prerelease) version of _Black_ is + released.\ + ℹ Recommended for users who want to preview or test alpha versions of _Black_. Note that + the most recent release may be newer than any prerelease, because no prereleases are created + before most releases. - `latest` - tag used for the newest image of _Black_.\ ℹ Recommended for users who always want to use the latest version of _Black_, even before it is released. diff --git a/pyproject.toml b/pyproject.toml index aede497e2af..ab38908ba15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ dependencies = [ "mypy_extensions>=0.4.3", "pathspec>=0.9.0", "platformdirs>=2", - "tomli>=1.1.0; python_full_version < '3.11.0a7'", + "tomli>=1.1.0; python_version < '3.11'", "typed-ast>=1.4.2; python_version < '3.8' and implementation_name == 'cpython'", "typing_extensions>=3.10.0.0; python_version < '3.10'", ] diff --git a/src/black/brackets.py b/src/black/brackets.py index ec9708cb08a..343f0608d50 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -80,9 +80,12 @@ def mark(self, leaf: Leaf) -> None: within brackets a given leaf is. 0 means there are no enclosing brackets that started on this line. - If a leaf is itself a closing bracket, it receives an `opening_bracket` - field that it forms a pair with. This is a one-directional link to - avoid reference cycles. + If a leaf is itself a closing bracket and there is a matching opening + bracket earlier, it receives an `opening_bracket` field with which it forms a + pair. This is a one-directional link to avoid reference cycles. Closing + bracket without opening happens on lines continued from previous + breaks, e.g. `) -> "ReturnType":` as part of a funcdef where we place + the return type annotation on its own line of the previous closing RPAR. If a leaf is a delimiter (a token on which Black can split the line if needed) and it's on depth 0, its `id()` is stored in the tracker's @@ -91,6 +94,13 @@ def mark(self, leaf: Leaf) -> None: if leaf.type == token.COMMENT: return + if ( + self.depth == 0 + and leaf.type in CLOSING_BRACKETS + and (self.depth, leaf.type) not in self.bracket_match + ): + return + self.maybe_decrement_after_for_loop_variable(leaf) self.maybe_decrement_after_lambda_arguments(leaf) if leaf.type in CLOSING_BRACKETS: diff --git a/src/black/linegen.py b/src/black/linegen.py index fe6ea11c501..2e75bc94506 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1268,6 +1268,8 @@ def maybe_make_parens_invisible_in_atom( syms.expr_stmt, syms.assert_stmt, syms.return_stmt, + syms.except_clause, + syms.funcdef, # these ones aren't useful to end users, but they do please fuzzers syms.for_stmt, syms.del_stmt, diff --git a/src/black/lines.py b/src/black/lines.py index 5e32b664cf0..b829a3a50f0 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -521,7 +521,8 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: and (self.semantic_leading_comment is None or before) ): self.semantic_leading_comment = block - elif not current_line.is_decorator: + # `or before` means this decorator already has an empty line before + elif not current_line.is_decorator or before: self.semantic_leading_comment = None self.previous_line = current_line diff --git a/src/black/nodes.py b/src/black/nodes.py index aeb2be389c8..a11fb7cc071 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -848,3 +848,15 @@ def is_string_token(nl: NL) -> TypeGuard[Leaf]: def is_number_token(nl: NL) -> TypeGuard[Leaf]: return nl.type == token.NUMBER + + +def is_part_of_annotation(leaf: Leaf) -> bool: + """Returns whether this leaf is part of type annotations.""" + ancestor = leaf.parent + while ancestor is not None: + if ancestor.prev_sibling and ancestor.prev_sibling.type == token.RARROW: + return True + if ancestor.parent and ancestor.parent.type == syms.tname: + return True + ancestor = ancestor.parent + return False diff --git a/src/black/trans.py b/src/black/trans.py index b78e139bd3c..3c7dfc74c02 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -30,7 +30,6 @@ from mypy_extensions import trait -from black.brackets import BracketMatchError from black.comments import contains_pragma_comment from black.lines import Line, append_leaves from black.mode import Feature @@ -41,6 +40,7 @@ is_empty_lpar, is_empty_par, is_empty_rpar, + is_part_of_annotation, parent_type, replace_child, syms, @@ -71,7 +71,7 @@ class CannotTransform(Exception): ParserState = int StringID = int TResult = Result[T, CannotTransform] # (T)ransform Result -TMatchResult = TResult[Index] +TMatchResult = TResult[List[Index]] SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops @@ -202,14 +202,19 @@ def __init__(self, line_length: int, normalize_strings: bool) -> None: def do_match(self, line: Line) -> TMatchResult: """ Returns: - * Ok(string_idx) such that `line.leaves[string_idx]` is our target - string, if a match was able to be made. + * Ok(string_indices) such that for each index, `line.leaves[index]` + is our target string if a match was able to be made. For + transformers that don't result in more lines (e.g. StringMerger, + StringParenStripper), multiple matches and transforms are done at + once to reduce the complexity. OR - * Err(CannotTransform), if a match was not able to be made. + * Err(CannotTransform), if no match could be made. """ @abstractmethod - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + def do_transform( + self, line: Line, string_indices: List[int] + ) -> Iterator[TResult[Line]]: """ Yields: * Ok(new_line) where new_line is the new transformed line. @@ -250,9 +255,9 @@ def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line] " this line as one that it can transform." ) from cant_transform - string_idx = match_result.ok() + string_indices = match_result.ok() - for line_result in self.do_transform(line, string_idx): + for line_result in self.do_transform(line, string_indices): if isinstance(line_result, Err): cant_transform = line_result.err() raise CannotTransform( @@ -355,7 +360,7 @@ class StringMerger(StringTransformer, CustomSplitMapMixin): Requirements: (A) The line contains adjacent strings such that ALL of the validation checks - listed in StringMerger.__validate_msg(...)'s docstring pass. + listed in StringMerger._validate_msg(...)'s docstring pass. OR (B) The line contains a string which uses line continuation backslashes. @@ -375,28 +380,50 @@ def do_match(self, line: Line) -> TMatchResult: is_valid_index = is_valid_index_factory(LL) - for i, leaf in enumerate(LL): + string_indices = [] + idx = 0 + while is_valid_index(idx): + leaf = LL[idx] if ( leaf.type == token.STRING - and is_valid_index(i + 1) - and LL[i + 1].type == token.STRING + and is_valid_index(idx + 1) + and LL[idx + 1].type == token.STRING ): - return Ok(i) + if not is_part_of_annotation(leaf): + string_indices.append(idx) - if leaf.type == token.STRING and "\\\n" in leaf.value: - return Ok(i) + # Advance to the next non-STRING leaf. + idx += 2 + while is_valid_index(idx) and LL[idx].type == token.STRING: + idx += 1 - return TErr("This line has no strings that need merging.") + elif leaf.type == token.STRING and "\\\n" in leaf.value: + string_indices.append(idx) + # Advance to the next non-STRING leaf. + idx += 1 + while is_valid_index(idx) and LL[idx].type == token.STRING: + idx += 1 - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + else: + idx += 1 + + if string_indices: + return Ok(string_indices) + else: + return TErr("This line has no strings that need merging.") + + def do_transform( + self, line: Line, string_indices: List[int] + ) -> Iterator[TResult[Line]]: new_line = line + rblc_result = self._remove_backslash_line_continuation_chars( - new_line, string_idx + new_line, string_indices ) if isinstance(rblc_result, Ok): new_line = rblc_result.ok() - msg_result = self._merge_string_group(new_line, string_idx) + msg_result = self._merge_string_group(new_line, string_indices) if isinstance(msg_result, Ok): new_line = msg_result.ok() @@ -417,7 +444,7 @@ def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: @staticmethod def _remove_backslash_line_continuation_chars( - line: Line, string_idx: int + line: Line, string_indices: List[int] ) -> TResult[Line]: """ Merge strings that were split across multiple lines using @@ -431,34 +458,44 @@ def _remove_backslash_line_continuation_chars( """ LL = line.leaves - string_leaf = LL[string_idx] - if not ( - string_leaf.type == token.STRING - and "\\\n" in string_leaf.value - and not has_triple_quotes(string_leaf.value) - ): + indices_to_transform = [] + for string_idx in string_indices: + string_leaf = LL[string_idx] + if ( + string_leaf.type == token.STRING + and "\\\n" in string_leaf.value + and not has_triple_quotes(string_leaf.value) + ): + indices_to_transform.append(string_idx) + + if not indices_to_transform: return TErr( - f"String leaf {string_leaf} does not contain any backslash line" - " continuation characters." + "Found no string leaves that contain backslash line continuation" + " characters." ) new_line = line.clone() new_line.comments = line.comments.copy() append_leaves(new_line, line, LL) - new_string_leaf = new_line.leaves[string_idx] - new_string_leaf.value = new_string_leaf.value.replace("\\\n", "") + for string_idx in indices_to_transform: + new_string_leaf = new_line.leaves[string_idx] + new_string_leaf.value = new_string_leaf.value.replace("\\\n", "") return Ok(new_line) - def _merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]: + def _merge_string_group( + self, line: Line, string_indices: List[int] + ) -> TResult[Line]: """ - Merges string group (i.e. set of adjacent strings) where the first - string in the group is `line.leaves[string_idx]`. + Merges string groups (i.e. set of adjacent strings). + + Each index from `string_indices` designates one string group's first + leaf in `line.leaves`. Returns: Ok(new_line), if ALL of the validation checks found in - __validate_msg(...) pass. + _validate_msg(...) pass. OR Err(CannotTransform), otherwise. """ @@ -466,10 +503,54 @@ def _merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]: is_valid_index = is_valid_index_factory(LL) - vresult = self._validate_msg(line, string_idx) - if isinstance(vresult, Err): - return vresult + # A dict of {string_idx: tuple[num_of_strings, string_leaf]}. + merged_string_idx_dict: Dict[int, Tuple[int, Leaf]] = {} + for string_idx in string_indices: + vresult = self._validate_msg(line, string_idx) + if isinstance(vresult, Err): + continue + merged_string_idx_dict[string_idx] = self._merge_one_string_group( + LL, string_idx, is_valid_index + ) + + if not merged_string_idx_dict: + return TErr("No string group is merged") + + # Build the final line ('new_line') that this method will later return. + new_line = line.clone() + previous_merged_string_idx = -1 + previous_merged_num_of_strings = -1 + for i, leaf in enumerate(LL): + if i in merged_string_idx_dict: + previous_merged_string_idx = i + previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i] + new_line.append(string_leaf) + + if ( + previous_merged_string_idx + <= i + < previous_merged_string_idx + previous_merged_num_of_strings + ): + for comment_leaf in line.comments_after(LL[i]): + new_line.append(comment_leaf, preformatted=True) + continue + + append_leaves(new_line, line, [leaf]) + + return Ok(new_line) + + def _merge_one_string_group( + self, LL: List[Leaf], string_idx: int, is_valid_index: Callable[[int], bool] + ) -> Tuple[int, Leaf]: + """ + Merges one string group where the first string in the group is + `LL[string_idx]`. + Returns: + A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the + number of strings merged and `leaf` is the newly merged string + to be replaced in the new line. + """ # If the string group is wrapped inside an Atom node, we must make sure # to later replace that Atom with our new (merged) string leaf. atom_node = LL[string_idx].parent @@ -592,27 +673,14 @@ def make_naked(string: str, string_prefix: str) -> str: # Else replace the atom node with the new string leaf. replace_child(atom_node, string_leaf) - # Build the final line ('new_line') that this method will later return. - new_line = line.clone() - for i, leaf in enumerate(LL): - if i == string_idx: - new_line.append(string_leaf) - - if string_idx <= i < string_idx + num_of_strings: - for comment_leaf in line.comments_after(LL[i]): - new_line.append(comment_leaf, preformatted=True) - continue - - append_leaves(new_line, line, [leaf]) - self.add_custom_splits(string_leaf.value, custom_splits) - return Ok(new_line) + return num_of_strings, string_leaf @staticmethod def _validate_msg(line: Line, string_idx: int) -> TResult[None]: """Validate (M)erge (S)tring (G)roup - Transform-time string validation logic for __merge_string_group(...). + Transform-time string validation logic for _merge_string_group(...). Returns: * Ok(None), if ALL validation checks (listed below) pass. @@ -626,6 +694,11 @@ def _validate_msg(line: Line, string_idx: int) -> TResult[None]: - The set of all string prefixes in the string group is of length greater than one and is not equal to {"", "f"}. - The string group consists of raw strings. + - The string group is stringified type annotations. We don't want to + process stringified type annotations since pyright doesn't support + them spanning multiple string values. (NOTE: mypy, pytype, pyre do + support them, so we can change if pyright also gains support in the + future. See https://github.com/microsoft/pyright/issues/4359.) """ # We first check for "inner" stand-alone comments (i.e. stand-alone # comments that have a string leaf before them AND after them). @@ -715,7 +788,15 @@ def do_match(self, line: Line) -> TMatchResult: is_valid_index = is_valid_index_factory(LL) - for idx, leaf in enumerate(LL): + string_indices = [] + + idx = -1 + while True: + idx += 1 + if idx >= len(LL): + break + leaf = LL[idx] + # Should be a string... if leaf.type != token.STRING: continue @@ -797,45 +878,73 @@ def do_match(self, line: Line) -> TMatchResult: }: continue - return Ok(string_idx) + string_indices.append(string_idx) + idx = string_idx + while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING: + idx += 1 + if string_indices: + return Ok(string_indices) return TErr("This line has no strings wrapped in parens.") - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + def do_transform( + self, line: Line, string_indices: List[int] + ) -> Iterator[TResult[Line]]: LL = line.leaves - string_parser = StringParser() - rpar_idx = string_parser.parse(LL, string_idx) + string_and_rpar_indices: List[int] = [] + for string_idx in string_indices: + string_parser = StringParser() + rpar_idx = string_parser.parse(LL, string_idx) + + should_transform = True + for leaf in (LL[string_idx - 1], LL[rpar_idx]): + if line.comments_after(leaf): + # Should not strip parentheses which have comments attached + # to them. + should_transform = False + break + if should_transform: + string_and_rpar_indices.extend((string_idx, rpar_idx)) - for leaf in (LL[string_idx - 1], LL[rpar_idx]): - if line.comments_after(leaf): - yield TErr( - "Will not strip parentheses which have comments attached to them." - ) - return + if string_and_rpar_indices: + yield Ok(self._transform_to_new_line(line, string_and_rpar_indices)) + else: + yield Err( + CannotTransform("All string groups have comments attached to them.") + ) + + def _transform_to_new_line( + self, line: Line, string_and_rpar_indices: List[int] + ) -> Line: + LL = line.leaves new_line = line.clone() new_line.comments = line.comments.copy() - try: - append_leaves(new_line, line, LL[: string_idx - 1]) - except BracketMatchError: - # HACK: I believe there is currently a bug somewhere in - # right_hand_split() that is causing brackets to not be tracked - # properly by a shared BracketTracker. - append_leaves(new_line, line, LL[: string_idx - 1], preformatted=True) - - string_leaf = Leaf(token.STRING, LL[string_idx].value) - LL[string_idx - 1].remove() - replace_child(LL[string_idx], string_leaf) - new_line.append(string_leaf) - - append_leaves( - new_line, line, LL[string_idx + 1 : rpar_idx] + LL[rpar_idx + 1 :] - ) - LL[rpar_idx].remove() + previous_idx = -1 + # We need to sort the indices, since string_idx and its matching + # rpar_idx may not come in order, e.g. in + # `("outer" % ("inner".join(items)))`, the "inner" string's + # string_idx is smaller than "outer" string's rpar_idx. + for idx in sorted(string_and_rpar_indices): + leaf = LL[idx] + lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx + append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx]) + if leaf.type == token.STRING: + string_leaf = Leaf(token.STRING, LL[idx].value) + LL[lpar_or_rpar_idx].remove() # Remove lpar. + replace_child(LL[idx], string_leaf) + new_line.append(string_leaf) + else: + LL[lpar_or_rpar_idx].remove() # This is a rpar. + + previous_idx = idx + + # Append the leaves after the last idx: + append_leaves(new_line, line, LL[idx + 1 :]) - yield Ok(new_line) + return new_line class BaseStringSplitter(StringTransformer): @@ -888,7 +997,12 @@ def do_match(self, line: Line) -> TMatchResult: if isinstance(match_result, Err): return match_result - string_idx = match_result.ok() + string_indices = match_result.ok() + assert len(string_indices) == 1, ( + f"{self.__class__.__name__} should only find one match at a time, found" + f" {len(string_indices)}" + ) + string_idx = string_indices[0] vresult = self._validate(line, string_idx) if isinstance(vresult, Err): return vresult @@ -1222,10 +1336,17 @@ def do_splitter_match(self, line: Line) -> TMatchResult: if is_valid_index(idx): return TErr("This line does not end with a string.") - return Ok(string_idx) + return Ok([string_idx]) - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + def do_transform( + self, line: Line, string_indices: List[int] + ) -> Iterator[TResult[Line]]: LL = line.leaves + assert len(string_indices) == 1, ( + f"{self.__class__.__name__} should only find one match at a time, found" + f" {len(string_indices)}" + ) + string_idx = string_indices[0] QUOTE = LL[string_idx].value[-1] @@ -1368,9 +1489,14 @@ def more_splits_should_be_made() -> bool: # prefix, and the current custom split did NOT originally use a # prefix... if ( - next_value != self._normalize_f_string(next_value, prefix) - and use_custom_breakpoints + use_custom_breakpoints and not csplit.has_prefix + and ( + # `next_value == prefix + QUOTE` happens when the custom + # split is an empty string. + next_value == prefix + QUOTE + or next_value != self._normalize_f_string(next_value, prefix) + ) ): # Then `csplit.break_idx` will be off by one after removing # the 'f' prefix. @@ -1716,7 +1842,7 @@ def do_splitter_match(self, line: Line) -> TMatchResult: " resultant line would still be over the specified line" " length and can't be split further by StringSplitter." ) - return Ok(string_idx) + return Ok([string_idx]) return TErr("This line does not contain any non-atomic strings.") @@ -1872,7 +1998,7 @@ def _dict_or_lambda_match(LL: List[Leaf]) -> Optional[int]: for i, leaf in enumerate(LL): # We MUST find a colon, it can either be dict's or lambda's colon... - if leaf.type == token.COLON: + if leaf.type == token.COLON and i < len(LL) - 1: idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 # That colon MUST be followed by a string... @@ -1893,8 +2019,15 @@ def _dict_or_lambda_match(LL: List[Leaf]) -> Optional[int]: return None - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + def do_transform( + self, line: Line, string_indices: List[int] + ) -> Iterator[TResult[Line]]: LL = line.leaves + assert len(string_indices) == 1, ( + f"{self.__class__.__name__} should only find one match at a time, found" + f" {len(string_indices)}" + ) + string_idx = string_indices[0] is_valid_index = is_valid_index_factory(LL) insert_str_child = insert_str_child_factory(LL[string_idx]) diff --git a/tests/data/preview/comments9.py b/tests/data/preview/comments9.py index 449612c037a..77b25556e74 100644 --- a/tests/data/preview/comments9.py +++ b/tests/data/preview/comments9.py @@ -114,6 +114,31 @@ def first_method(self): pass +# Regression test for https://github.com/psf/black/issues/3454. +def foo(): + pass + # Trailing comment that belongs to this function + + +@decorator1 +@decorator2 # fmt: skip +def bar(): + pass + + +# Regression test for https://github.com/psf/black/issues/3454. +def foo(): + pass + # Trailing comment that belongs to this function. + # NOTE this comment only has one empty line below, and the formatter + # should enforce two blank lines. + +@decorator1 +# A standalone comment +def bar(): + pass + + # output @@ -252,3 +277,29 @@ class MyClass: # More comments. def first_method(self): pass + + +# Regression test for https://github.com/psf/black/issues/3454. +def foo(): + pass + # Trailing comment that belongs to this function + + +@decorator1 +@decorator2 # fmt: skip +def bar(): + pass + + +# Regression test for https://github.com/psf/black/issues/3454. +def foo(): + pass + # Trailing comment that belongs to this function. + # NOTE this comment only has one empty line below, and the formatter + # should enforce two blank lines. + + +@decorator1 +# A standalone comment +def bar(): + pass diff --git a/tests/data/preview/long_strings.py b/tests/data/preview/long_strings.py index 9c78f675b8f..b7a0a42f82a 100644 --- a/tests/data/preview/long_strings.py +++ b/tests/data/preview/long_strings.py @@ -287,6 +287,23 @@ def foo(): ), } +# Complex string concatenations with a method call in the middle. +code = ( + (" return [\n") + + ( + ", \n".join( + " (%r, self.%s, visitor.%s)" + % (attrname, attrname, visit_name) + for attrname, visit_name in names + ) + ) + + ("\n ]\n") +) + + +# Test case of an outer string' parens enclose an inner string's parens. +call(body=("%s %s" % ((",".join(items)), suffix))) + # output @@ -828,3 +845,17 @@ def foo(): f"{some_function_call(j.right)})" ), } + +# Complex string concatenations with a method call in the middle. +code = ( + " return [\n" + + ", \n".join( + " (%r, self.%s, visitor.%s)" % (attrname, attrname, visit_name) + for attrname, visit_name in names + ) + + "\n ]\n" +) + + +# Test case of an outer string' parens enclose an inner string's parens. +call(body="%s %s" % (",".join(items), suffix)) diff --git a/tests/data/preview/long_strings__regression.py b/tests/data/preview/long_strings__regression.py index 8b8fc179147..ef9007f4ce1 100644 --- a/tests/data/preview/long_strings__regression.py +++ b/tests/data/preview/long_strings__regression.py @@ -531,6 +531,25 @@ async def foo(self): r"signiferumque, duo ea vocibus consetetur scriptorem. Facer \t", } +# Regression test for https://github.com/psf/black/issues/3459. +xxxx( + empty_str_as_first_split='' + f'xxxxxxx {xxxxxxxxxx} xxx xxxxxxxxxx xxxxx xxx xxx xx ' + 'xxxxx xxxxxxxxx xxxxxxx, xxx xxxxxxxxxxx xxx xxxxx. ' + f'xxxxxxxxxxxxx xxxx xx xxxxxxxxxx. xxxxx: {x.xxx}', + empty_u_str_as_first_split=u'' + f'xxxxxxx {xxxxxxxxxx} xxx xxxxxxxxxx xxxxx xxx xxx xx ' + 'xxxxx xxxxxxxxx xxxxxxx, xxx xxxxxxxxxxx xxx xxxxx. ' + f'xxxxxxxxxxxxx xxxx xx xxxxxxxxxx. xxxxx: {x.xxx}', +) + +# Regression test for https://github.com/psf/black/issues/3455. +a_dict = { + "/this/is/a/very/very/very/very/very/very/very/very/very/very/long/key/without/spaces": + # And there is a comment before the value + ("item1", "item2", "item3"), +} + # output @@ -1193,3 +1212,26 @@ async def foo(self): r"signiferumque, duo ea vocibus consetetur scriptorem. Facer \t" ), } + +# Regression test for https://github.com/psf/black/issues/3459. +xxxx( + empty_str_as_first_split=( + "" + f"xxxxxxx {xxxxxxxxxx} xxx xxxxxxxxxx xxxxx xxx xxx xx " + "xxxxx xxxxxxxxx xxxxxxx, xxx xxxxxxxxxxx xxx xxxxx. " + f"xxxxxxxxxxxxx xxxx xx xxxxxxxxxx. xxxxx: {x.xxx}" + ), + empty_u_str_as_first_split=( + "" + f"xxxxxxx {xxxxxxxxxx} xxx xxxxxxxxxx xxxxx xxx xxx xx " + "xxxxx xxxxxxxxx xxxxxxx, xxx xxxxxxxxxxx xxx xxxxx. " + f"xxxxxxxxxxxxx xxxx xx xxxxxxxxxx. xxxxx: {x.xxx}" + ), +) + +# Regression test for https://github.com/psf/black/issues/3455. +a_dict = { + "/this/is/a/very/very/very/very/very/very/very/very/very/very/long/key/without/spaces": + # And there is a comment before the value + ("item1", "item2", "item3"), +} diff --git a/tests/data/preview/long_strings__type_annotations.py b/tests/data/preview/long_strings__type_annotations.py new file mode 100644 index 00000000000..41d7ee2b67b --- /dev/null +++ b/tests/data/preview/long_strings__type_annotations.py @@ -0,0 +1,59 @@ +def func( + arg1, + arg2, +) -> Set["this_is_a_very_long_module_name.AndAVeryLongClasName" + ".WithAVeryVeryVeryVeryVeryLongSubClassName"]: + pass + + +def func( + argument: ( + "VeryLongClassNameWithAwkwardGenericSubtype[int] |" + "VeryLongClassNameWithAwkwardGenericSubtype[str]" + ), +) -> ( + "VeryLongClassNameWithAwkwardGenericSubtype[int] |" + "VeryLongClassNameWithAwkwardGenericSubtype[str]" +): + pass + + +def func( + argument: ( + "int |" + "str" + ), +) -> Set["int |" + " str"]: + pass + + +# output + + +def func( + arg1, + arg2, +) -> Set[ + "this_is_a_very_long_module_name.AndAVeryLongClasName" + ".WithAVeryVeryVeryVeryVeryLongSubClassName" +]: + pass + + +def func( + argument: ( + "VeryLongClassNameWithAwkwardGenericSubtype[int] |" + "VeryLongClassNameWithAwkwardGenericSubtype[str]" + ), +) -> ( + "VeryLongClassNameWithAwkwardGenericSubtype[int] |" + "VeryLongClassNameWithAwkwardGenericSubtype[str]" +): + pass + + +def func( + argument: ("int |" "str"), +) -> Set["int |" " str"]: + pass diff --git a/tests/data/py_310/pattern_matching_extras.py b/tests/data/py_310/pattern_matching_extras.py index 9f6907f7575..0242d264e5b 100644 --- a/tests/data/py_310/pattern_matching_extras.py +++ b/tests/data/py_310/pattern_matching_extras.py @@ -114,6 +114,6 @@ def func(match: case, case: match) -> case: match bar1: case Foo( - normal=x, perhaps=[list, {an: d, dict: 1.0}] as y, otherwise=something, q=t as u + normal=x, perhaps=[list, {"x": d, "y": 1.0}] as y, otherwise=something, q=t as u ): pass diff --git a/tests/util.py b/tests/util.py index d65c2e651ae..967d576fafe 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,6 +2,7 @@ import sys import unittest from contextlib import contextmanager +from dataclasses import replace from functools import partial from pathlib import Path from typing import Any, Iterator, List, Optional, Tuple @@ -56,6 +57,10 @@ def _assert_format_equal(expected: str, actual: str) -> None: assert actual == expected +class FormatFailure(Exception): + """Used to wrap failures when assert_format() runs in an extra mode.""" + + def assert_format( source: str, expected: str, @@ -70,12 +75,57 @@ def assert_format( safety guards so they don't just crash with a SyntaxError. Please note this is separate from TargetVerson Mode configuration. """ + _assert_format_inner( + source, expected, mode, fast=fast, minimum_version=minimum_version + ) + + # For both preview and non-preview tests, ensure that Black doesn't crash on + # this code, but don't pass "expected" because the precise output may differ. + try: + _assert_format_inner( + source, + None, + replace(mode, preview=not mode.preview), + fast=fast, + minimum_version=minimum_version, + ) + except Exception as e: + text = "non-preview" if mode.preview else "preview" + raise FormatFailure( + f"Black crashed formatting this case in {text} mode." + ) from e + # Similarly, setting line length to 1 is a good way to catch + # stability bugs. But only in non-preview mode because preview mode + # currently has a lot of line length 1 bugs. + try: + _assert_format_inner( + source, + None, + replace(mode, preview=False, line_length=1), + fast=fast, + minimum_version=minimum_version, + ) + except Exception as e: + raise FormatFailure( + "Black crashed formatting this case with line-length set to 1." + ) from e + + +def _assert_format_inner( + source: str, + expected: Optional[str] = None, + mode: black.Mode = DEFAULT_MODE, + *, + fast: bool = False, + minimum_version: Optional[Tuple[int, int]] = None, +) -> None: actual = black.format_str(source, mode=mode) - _assert_format_equal(expected, actual) + if expected is not None: + _assert_format_equal(expected, actual) # It's not useful to run safety checks if we're expecting no changes anyway. The # assertion right above will raise if reality does actually make changes. This just # avoids wasted CPU cycles. - if not fast and source != expected: + if not fast and source != actual: # Unfortunately the AST equivalence check relies on the built-in ast module # being able to parse the code being formatted. This doesn't always work out # when checking modern code on older versions.