Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for all valid string literals #115

Merged
merged 5 commits into from Apr 9, 2018
Merged

Add support for all valid string literals #115

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1f2858d
Tokenize only valid string prefixes
zsol Apr 6, 2018
2c5fe2e
Add test case for rf-strings
zsol Apr 6, 2018
c8fe2a4
mention string support in blib2to3/README
zsol Apr 6, 2018
8bcf2cd
add support for `ur` prefix from py2
zsol Apr 9, 2018
8b7bd90
Notation nits
ambv Apr 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 4 additions & 4 deletions black.py
View file
Open in desktop
Expand Up @@ -1923,8 +1923,8 @@ def normalize_string_quotes(leaf: Leaf) -> None:

prefix = leaf.value[:first_quote_pos]
body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)]
unescaped_new_quote = re.compile(r"(([^\\]|^)(\\\\)*)" + new_quote)
escaped_orig_quote = re.compile(r"\\(\\\\)*" + orig_quote)
unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
escaped_orig_quote = re.compile(rf"\\(\\\\)*{orig_quote}")
if "r" in prefix.casefold():
if unescaped_new_quote.search(body):
# There's at least one unescaped new_quote in this raw string
Expand All @@ -1934,8 +1934,8 @@ def normalize_string_quotes(leaf: Leaf) -> None:
# Do not introduce or remove backslashes in raw strings
new_body = body
else:
new_body = escaped_orig_quote.sub(f"\\1{orig_quote}", body)
new_body = unescaped_new_quote.sub(f"\\1\\\\{new_quote}", new_body)
new_body = escaped_orig_quote.sub(rf"\1{orig_quote}", body)
new_body = unescaped_new_quote.sub(rf"\1\\{new_quote}", new_body)
if new_quote == '"""' and new_body[-1] == '"':
# edge case:
new_body = new_body[:-1] + '\\"'
Expand Down
1 change: 1 addition & 0 deletions blib2to3/README
View file
Open in desktop
Expand Up @@ -7,6 +7,7 @@ Reasons for forking:
*args and **kwargs
- backport of GH-6143 that restores the ability to reformat legacy usage of
`async`
- support all types of string literals
- better ability to debug (better reprs)
- INDENT and DEDENT don't hold whitespace and comment prefixes
- ability to Cythonize
74 changes: 22 additions & 52 deletions blib2to3/pgen2/tokenize.py
View file
Open in desktop
Expand Up @@ -48,6 +48,8 @@
def group(*choices): return '(' + '|'.join(choices) + ')'
def any(*choices): return group(*choices) + '*'
def maybe(*choices): return group(*choices) + '?'
def combinations(*l):
return [(x + y).strip() for x in l for y in l + (" ",) if x.casefold() != y.casefold()]

Whitespace = r'[ \f\t]*'
Comment = r'#[^\r\n]*'
Expand All @@ -74,7 +76,7 @@ def maybe(*choices): return group(*choices) + '?'
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
# Tail end of """ string.
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?"
_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbB][rR])?"
Triple = group(_litprefix + "'''", _litprefix + '"""')
# Single-line ' or " string.
String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
Expand Down Expand Up @@ -107,59 +109,27 @@ def maybe(*choices): return group(*choices) + '?'
pseudoprog = re.compile(PseudoToken, re.UNICODE)
single3prog = re.compile(Single3)
double3prog = re.compile(Double3)

_strprefixes = set(combinations('r', 'R', 'f', 'F') +
combinations('r', 'R', 'b', 'B') +
['u', 'U'])
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're missing 'uR', 'UR', 'Ur', 'ur' which is valid on Python 2.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uR right 🤣 I was wondering why tokenize used to accept this in the regex above.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Funnily enough, since we are still supporting u"" in Python 3, not supporting ur"" is likely just an omission of whoever brought u"" back :-)


endprogs = {"'": re.compile(Single), '"': re.compile(Double),
"'''": single3prog, '"""': double3prog,
"r'''": single3prog, 'r"""': double3prog,
"u'''": single3prog, 'u"""': double3prog,
"b'''": single3prog, 'b"""': double3prog,
"f'''": single3prog, 'f"""': double3prog,
"ur'''": single3prog, 'ur"""': double3prog,
"br'''": single3prog, 'br"""': double3prog,
"rb'''": single3prog, 'rb"""': double3prog,
"R'''": single3prog, 'R"""': double3prog,
"U'''": single3prog, 'U"""': double3prog,
"B'''": single3prog, 'B"""': double3prog,
"F'''": single3prog, 'F"""': double3prog,
"uR'''": single3prog, 'uR"""': double3prog,
"Ur'''": single3prog, 'Ur"""': double3prog,
"UR'''": single3prog, 'UR"""': double3prog,
"bR'''": single3prog, 'bR"""': double3prog,
"Br'''": single3prog, 'Br"""': double3prog,
"BR'''": single3prog, 'BR"""': double3prog,
"rB'''": single3prog, 'rB"""': double3prog,
"Rb'''": single3prog, 'Rb"""': double3prog,
"RB'''": single3prog, 'RB"""': double3prog,
'r': None, 'R': None,
'u': None, 'U': None,
'f': None, 'F': None,
'b': None, 'B': None}

triple_quoted = {}
for t in ("'''", '"""',
"r'''", 'r"""', "R'''", 'R"""',
"u'''", 'u"""', "U'''", 'U"""',
"b'''", 'b"""', "B'''", 'B"""',
"f'''", 'f"""', "F'''", 'F"""',
"ur'''", 'ur"""', "Ur'''", 'Ur"""',
"uR'''", 'uR"""', "UR'''", 'UR"""',
"br'''", 'br"""', "Br'''", 'Br"""',
"bR'''", 'bR"""', "BR'''", 'BR"""',
"rb'''", 'rb"""', "Rb'''", 'Rb"""',
"rB'''", 'rB"""', "RB'''", 'RB"""',):
triple_quoted[t] = t
single_quoted = {}
for t in ("'", '"',
"r'", 'r"', "R'", 'R"',
"u'", 'u"', "U'", 'U"',
"b'", 'b"', "B'", 'B"',
"f'", 'f"', "F'", 'F"',
"ur'", 'ur"', "Ur'", 'Ur"',
"uR'", 'uR"', "UR'", 'UR"',
"br'", 'br"', "Br'", 'Br"',
"bR'", 'bR"', "BR'", 'BR"',
"rb'", 'rb"', "Rb'", 'Rb"',
"rB'", 'rB"', "RB'", 'RB"',):
single_quoted[t] = t
**{f"{prefix}'''": single3prog for prefix in _strprefixes},
**{f'{prefix}"""': double3prog for prefix in _strprefixes},
**{prefix: None for prefix in _strprefixes}}

triple_quoted = (
{"'''", '"""'} |
{f"{prefix}'''" for prefix in _strprefixes} |
{f'{prefix}"""' for prefix in _strprefixes}
)
single_quoted = (
{"'", '"'} |
{f"{prefix}'" for prefix in _strprefixes} |
{f'{prefix}"' for prefix in _strprefixes}
)

tabsize = 8

Expand Down
2 changes: 2 additions & 0 deletions tests/string_quotes.py
View file
Open in desktop
Expand Up @@ -22,6 +22,7 @@
r'Date d\'expiration:(.*)'
r'Tricky "quote'
r'Not-so-tricky \"quote'
rf'{yay}'
'\n\
The \"quick\"\n\
brown fox\n\
Expand Down Expand Up @@ -56,6 +57,7 @@
r"Date d\'expiration:(.*)"
r'Tricky "quote'
r"Not-so-tricky \"quote"
rf"{yay}"
"\n\
The \"quick\"\n\
brown fox\n\
Expand Down