Backport improvements

scrapy/downloadermiddlewares/httpcompression.py

@@ -39,6 +39,8 @@ class HttpCompressionMiddleware(object):
 
     def __init__(self, crawler=None):
         if not crawler:
+            self._max_size = 1073741824
+            self._warn_size = 33554432
             return
         self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
         self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")

scrapy/utils/_compression.py

@@ -12,6 +12,9 @@ except ImportError:
     pass
 
 
+_CHUNK_SIZE = 65536  # 64 KiB
+
+
 class _DecompressionMaxSizeExceeded(ValueError):
     pass
 
@@ -20,12 +23,11 @@ def _inflate(data, max_size=0):
     decompressor = zlib.decompressobj()
     raw_decompressor = zlib.decompressobj(-15)
     input_stream = BytesIO(data)
-    output_list = []
+    output_stream = BytesIO()
     output_chunk = b"."
     decompressed_size = 0
-    CHUNK_SIZE = 8196
     while output_chunk:
-        input_chunk = input_stream.read(CHUNK_SIZE)
+        input_chunk = input_stream.read(_CHUNK_SIZE)
         try:
             output_chunk = decompressor.decompress(input_chunk)
         except zlib.error:
@@ -49,19 +51,19 @@ def _inflate(data, max_size=0):
                     max_size=max_size,
                 )
             )
-        output_list.append(output_chunk)
-    return b"".join(output_list)
+        output_stream.write(output_chunk)
+    output_stream.seek(0)
+    return output_stream.read()
 
 
 def _unbrotli(data, max_size=0):
     decompressor = brotli.Decompressor()
     input_stream = BytesIO(data)
-    output_list = []
+    output_stream = BytesIO()
     output_chunk = b"."
     decompressed_size = 0
-    CHUNK_SIZE = 8196
     while output_chunk:
-        input_chunk = input_stream.read(CHUNK_SIZE)
+        input_chunk = input_stream.read(_CHUNK_SIZE)
         output_chunk = decompressor.decompress(input_chunk)
         decompressed_size += len(output_chunk)
         if max_size and decompressed_size > max_size:
@@ -73,19 +75,19 @@ def _unbrotli(data, max_size=0):
                     max_size=max_size,
                 )
             )
-        output_list.append(output_chunk)
-    return b"".join(output_list)
+        output_stream.write(output_chunk)
+    output_stream.seek(0)
+    return output_stream.read()
 
 
 def _unzstd(data, max_size=0):
     decompressor = zstandard.ZstdDecompressor()
     stream_reader = decompressor.stream_reader(BytesIO(data))
-    output_list = []
+    output_stream = BytesIO()
     output_chunk = b"."
     decompressed_size = 0
-    CHUNK_SIZE = 8196
     while output_chunk:
-        output_chunk = stream_reader.read(CHUNK_SIZE)
+        output_chunk = stream_reader.read(_CHUNK_SIZE)
         decompressed_size += len(output_chunk)
         if max_size and decompressed_size > max_size:
             raise _DecompressionMaxSizeExceeded(
@@ -96,5 +98,6 @@ def _unzstd(data, max_size=0):
                     max_size=max_size,
                 )
             )
-        output_list.append(output_chunk)
-    return b"".join(output_list)
+        output_stream.write(output_chunk)
+    output_stream.seek(0)
+    return output_stream.read()

scrapy/utils/gz.py

@@ -12,7 +12,7 @@ import six
 
 from scrapy.utils.decorators import deprecated
 
-from ._compression import _DecompressionMaxSizeExceeded
+from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
 
 # - Python>=3.5 GzipFile's read() has issues returning leftover
 #   uncompressed data when input is corrupted
@@ -35,25 +35,25 @@ def gunzip(data, max_size=0):
     This is resilient to CRC checksum errors.
     """
     f = GzipFile(fileobj=BytesIO(data))
-    output_list = []
-    chunk = b"."
+    output_stream = BytesIO()
+    output_chunk = b"."
     decompressed_size = 0
-    while chunk:
+    while output_chunk:
         try:
-            chunk = read1(f, 8196)
+            output_chunk = read1(f, _CHUNK_SIZE)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
             # some pages are quite small so output_list is empty and f.extrabuf
             # contains the whole page content
-            if output_list or getattr(f, 'extrabuf', None):
+            if decompressed_size or getattr(f, 'extrabuf', None):
                 try:
-                    output_list.append(f.extrabuf[-f.extrasize:])
+                    output_stream.write(f.extrabuf[-f.extrasize:])
                 finally:
                     break
             else:
                 raise
-        decompressed_size += len(chunk)
+        decompressed_size += len(output_chunk)
         if max_size and decompressed_size > max_size:
             raise _DecompressionMaxSizeExceeded(
                 "The number of bytes decompressed so far "
@@ -63,8 +63,9 @@ def gunzip(data, max_size=0):
                     max_size=max_size,
                 )
             )
-        output_list.append(chunk)
-    return b"".join(output_list)
+        output_stream.write(output_chunk)
+    output_stream.seek(0)
+    return output_stream.read()
 
 _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
 _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search