Add typing to scrapy/utils/gz.py and remove dead code.

This commit is contained in:
Andrey Rakhmatullin 2023-05-07 02:02:33 +04:00
parent 43ee483a0d
commit 4da8691510
1 changed files with 9 additions and 10 deletions

View File

@ -1,15 +1,18 @@
import struct
from gzip import GzipFile
from io import BytesIO
from typing import List
from scrapy.http import Response
def gunzip(data):
def gunzip(data: bytes) -> bytes:
"""Gunzip the given data and return as much data as possible.
This is resilient to CRC checksum errors.
"""
f = GzipFile(fileobj=BytesIO(data))
output_list = []
output_list: List[bytes] = []
chunk = b"."
while chunk:
try:
@ -18,17 +21,13 @@ def gunzip(data):
except (IOError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error
# some pages are quite small so output_list is empty and f.extrabuf
# contains the whole page content
if output_list or getattr(f, "extrabuf", None):
try:
output_list.append(f.extrabuf[-f.extrasize :])
finally:
break
# some pages are quite small so output_list is empty
if output_list:
break
else:
raise
return b"".join(output_list)
def gzip_magic_number(response):
def gzip_magic_number(response: Response) -> bool:
return response.body[:3] == b"\x1f\x8b\x08"