RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CHUNK_SIZE: 8 KiB → 32 KiB

This commit is contained in:
Adrián Chaves 2023-11-24 10:20:14 +01:00
parent 6c278e1862
commit 62398e424c
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/utils/_compression.py
View File

@ -12,6 +12,9 @@ except ImportError:
pass
_CHUNK_SIZE = 65536 # 64 KiB
class _DecompressionMaxSizeExceeded(ValueError):
pass
@ -23,9 +26,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
CHUNK_SIZE = 8196
while output_chunk:
input_chunk = input_stream.read(CHUNK_SIZE)
input_chunk = input_stream.read(_CHUNK_SIZE)
try:
output_chunk = decompressor.decompress(input_chunk)
except zlib.error:
@ -57,9 +59,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
CHUNK_SIZE = 8196
while output_chunk:
input_chunk = input_stream.read(CHUNK_SIZE)
input_chunk = input_stream.read(_CHUNK_SIZE)
output_chunk = decompressor.process(input_chunk)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
@ -79,9 +80,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:
output_stream = BytesIO()
output_chunk = b"."
decompressed_size = 0
CHUNK_SIZE = 8196
while output_chunk:
output_chunk = stream_reader.read(CHUNK_SIZE)
output_chunk = stream_reader.read(_CHUNK_SIZE)
decompressed_size += len(output_chunk)
if max_size and decompressed_size > max_size:
raise _DecompressionMaxSizeExceeded(

4
scrapy/utils/gz.py
View File

@ -4,7 +4,7 @@ from io import BytesIO
from scrapy.http import Response
from ._compression import _DecompressionMaxSizeExceeded
from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
@ -18,7 +18,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
decompressed_size = 0
while chunk:
try:
chunk = f.read1(8196)
chunk = f.read1(_CHUNK_SIZE)
except (OSError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error