Skip to content

Commit

Permalink
Replace quadratic algo in LineDecoder
Browse files Browse the repository at this point in the history
Leading to enormous speedups when doing things such as
Response(...).iter_lines() as described on issue #2422
  • Loading branch information
giannitedesco committed Oct 26, 2022
1 parent 9e97d7d commit 50c6811
Showing 1 changed file with 15 additions and 39 deletions.
54 changes: 15 additions & 39 deletions httpx/_decoders.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"""
import codecs
import io
import re
import typing
import zlib

Expand Down Expand Up @@ -262,49 +263,24 @@ class LineDecoder:
as line endings, normalizing to `\n`.
"""

_re = re.compile(r"(.*\n?)")

def __init__(self) -> None:
self.buffer = ""

def decode(self, text: str) -> typing.List[str]:
lines = []

if text and self.buffer and self.buffer[-1] == "\r":
if text.startswith("\n"):
# Handle the case where we have an "\r\n" split across
# our previous input, and our new chunk.
lines.append(self.buffer[:-1] + "\n")
self.buffer = ""
text = text[1:]
else:
# Handle the case where we have "\r" at the end of our
# previous input.
lines.append(self.buffer[:-1] + "\n")
self.buffer = ""

while text:
num_chars = len(text)
for idx in range(num_chars):
char = text[idx]
next_char = None if idx + 1 == num_chars else text[idx + 1]
if char == "\n":
lines.append(self.buffer + text[: idx + 1])
self.buffer = ""
text = text[idx + 1 :]
break
elif char == "\r" and next_char == "\n":
lines.append(self.buffer + text[:idx] + "\n")
self.buffer = ""
text = text[idx + 2 :]
break
elif char == "\r" and next_char is not None:
lines.append(self.buffer + text[:idx] + "\n")
self.buffer = ""
text = text[idx + 1 :]
break
elif next_char is None:
self.buffer += text
text = ""
break
if self.buffer:
text = self.buffer + text

lines = self._re.findall(text)
lines.pop() # always an empty match at the end
remainder = ""
if lines:
final = lines[-1]
if not final.endswith("\n"):
remainder = lines.pop()

self.buffer = remainder

return lines

Expand Down

0 comments on commit 50c6811

Please sign in to comment.