mirror of https://github.com/scrapy/scrapy.git
Add typing to scrapy/utils/gz.py and remove dead code.
This commit is contained in:
parent
43ee483a0d
commit
4da8691510
|
|
@ -1,15 +1,18 @@
|
|||
import struct
|
||||
from gzip import GzipFile
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
from scrapy.http import Response
|
||||
|
||||
|
||||
def gunzip(data):
|
||||
def gunzip(data: bytes) -> bytes:
|
||||
"""Gunzip the given data and return as much data as possible.
|
||||
|
||||
This is resilient to CRC checksum errors.
|
||||
"""
|
||||
f = GzipFile(fileobj=BytesIO(data))
|
||||
output_list = []
|
||||
output_list: List[bytes] = []
|
||||
chunk = b"."
|
||||
while chunk:
|
||||
try:
|
||||
|
|
@ -18,17 +21,13 @@ def gunzip(data):
|
|||
except (IOError, EOFError, struct.error):
|
||||
# complete only if there is some data, otherwise re-raise
|
||||
# see issue 87 about catching struct.error
|
||||
# some pages are quite small so output_list is empty and f.extrabuf
|
||||
# contains the whole page content
|
||||
if output_list or getattr(f, "extrabuf", None):
|
||||
try:
|
||||
output_list.append(f.extrabuf[-f.extrasize :])
|
||||
finally:
|
||||
break
|
||||
# some pages are quite small so output_list is empty
|
||||
if output_list:
|
||||
break
|
||||
else:
|
||||
raise
|
||||
return b"".join(output_list)
|
||||
|
||||
|
||||
def gzip_magic_number(response):
|
||||
def gzip_magic_number(response: Response) -> bool:
|
||||
return response.body[:3] == b"\x1f\x8b\x08"
|
||||
|
|
|
|||
Loading…
Reference in New Issue