mirror of https://github.com/scrapy/scrapy.git
Use if Py2/Py3 function instead of custom GzipFile class method
This commit is contained in:
parent
fd99ef86df
commit
1f2233837a
|
|
@ -7,24 +7,35 @@ except ImportError:
|
|||
from io import UnsupportedOperation
|
||||
from gzip import GzipFile
|
||||
|
||||
class ReadOneGzipFile(GzipFile):
|
||||
def readone(self, size=-1):
|
||||
try:
|
||||
return self.read1(size)
|
||||
except UnsupportedOperation:
|
||||
return self.read(size)
|
||||
import six
|
||||
|
||||
|
||||
# - Python>=3.5 GzipFile's read() has issues returning leftover
|
||||
# uncompressed data when input is corrupted
|
||||
# (regression or bug-fix compared to Python 3.4)
|
||||
# - read1(), which fetches data before raising EOFError on next call
|
||||
# works here but is only available from Python>=3.3
|
||||
# - scrapy does not support Python 3.2
|
||||
# - Python 2.7 GzipFile works fine with standard read() + extrabuf
|
||||
if six.PY3:
|
||||
def read1(gzf, size=-1):
|
||||
return gzf.read1(size)
|
||||
else:
|
||||
def read1(gzf, size=-1):
|
||||
return gzf.read(size)
|
||||
|
||||
|
||||
def gunzip(data):
|
||||
"""Gunzip the given data and return as much data as possible.
|
||||
|
||||
This is resilient to CRC checksum errors.
|
||||
"""
|
||||
f = ReadOneGzipFile(fileobj=BytesIO(data))
|
||||
f = GzipFile(fileobj=BytesIO(data))
|
||||
output = b''
|
||||
chunk = b'.'
|
||||
while chunk:
|
||||
try:
|
||||
chunk = f.readone(8196)
|
||||
chunk = read1(f, 8196)
|
||||
output += chunk
|
||||
except (IOError, EOFError, struct.error):
|
||||
# complete only if there is some data, otherwise re-raise
|
||||
|
|
|
|||
Loading…
Reference in New Issue