diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6db817ffe..f495f99a9 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -39,6 +39,8 @@ class HttpCompressionMiddleware(object): def __init__(self, crawler=None): if not crawler: + self._max_size = 1073741824 + self._warn_size = 33554432 return self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 2260c55a8..8b1f073bb 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -12,6 +12,9 @@ except ImportError: pass +_CHUNK_SIZE = 65536 # 64 KiB + + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -20,12 +23,11 @@ def _inflate(data, max_size=0): decompressor = zlib.decompressobj() raw_decompressor = zlib.decompressobj(-15) input_stream = BytesIO(data) - output_list = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - input_chunk = input_stream.read(CHUNK_SIZE) + input_chunk = input_stream.read(_CHUNK_SIZE) try: output_chunk = decompressor.decompress(input_chunk) except zlib.error: @@ -49,19 +51,19 @@ def _inflate(data, max_size=0): max_size=max_size, ) ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() def _unbrotli(data, max_size=0): decompressor = brotli.Decompressor() input_stream = BytesIO(data) - output_list = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - input_chunk = input_stream.read(CHUNK_SIZE) + input_chunk = input_stream.read(_CHUNK_SIZE) output_chunk = decompressor.decompress(input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: @@ -73,19 +75,19 @@ def _unbrotli(data, max_size=0): max_size=max_size, ) ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() def _unzstd(data, max_size=0): decompressor = zstandard.ZstdDecompressor() stream_reader = decompressor.stream_reader(BytesIO(data)) - output_list = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - output_chunk = stream_reader.read(CHUNK_SIZE) + output_chunk = stream_reader.read(_CHUNK_SIZE) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( @@ -96,5 +98,6 @@ def _unzstd(data, max_size=0): max_size=max_size, ) ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 9f28cfb77..15a1ecfe6 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -12,7 +12,7 @@ import six from scrapy.utils.decorators import deprecated -from ._compression import _DecompressionMaxSizeExceeded +from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded # - Python>=3.5 GzipFile's read() has issues returning leftover # uncompressed data when input is corrupted @@ -35,25 +35,25 @@ def gunzip(data, max_size=0): This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list = [] - chunk = b"." + output_stream = BytesIO() + output_chunk = b"." decompressed_size = 0 - while chunk: + while output_chunk: try: - chunk = read1(f, 8196) + output_chunk = read1(f, _CHUNK_SIZE) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output_list or getattr(f, 'extrabuf', None): + if decompressed_size or getattr(f, 'extrabuf', None): try: - output_list.append(f.extrabuf[-f.extrasize:]) + output_stream.write(f.extrabuf[-f.extrasize:]) finally: break else: raise - decompressed_size += len(chunk) + decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( "The number of bytes decompressed so far " @@ -63,8 +63,9 @@ def gunzip(data, max_size=0): max_size=max_size, ) ) - output_list.append(chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search