diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..98ca510ed 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,15 +1,18 @@ import struct from gzip import GzipFile from io import BytesIO +from typing import List + +from scrapy.http import Response -def gunzip(data): +def gunzip(data: bytes) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list = [] + output_list: List[bytes] = [] chunk = b"." while chunk: try: @@ -18,17 +21,13 @@ def gunzip(data): except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty and f.extrabuf - # contains the whole page content - if output_list or getattr(f, "extrabuf", None): - try: - output_list.append(f.extrabuf[-f.extrasize :]) - finally: - break + # some pages are quite small so output_list is empty + if output_list: + break else: raise return b"".join(output_list) -def gzip_magic_number(response): +def gzip_magic_number(response: Response) -> bool: return response.body[:3] == b"\x1f\x8b\x08"