Use if Py2/Py3 function instead of custom GzipFile class method

This commit is contained in:
Paul Tremberth 2016-01-19 16:58:24 +01:00
parent fd99ef86df
commit 1f2233837a
1 changed files with 19 additions and 8 deletions

View File

@ -7,24 +7,35 @@ except ImportError:
from io import UnsupportedOperation
from gzip import GzipFile
class ReadOneGzipFile(GzipFile):
def readone(self, size=-1):
try:
return self.read1(size)
except UnsupportedOperation:
return self.read(size)
import six
# - Python>=3.5 GzipFile's read() has issues returning leftover
# uncompressed data when input is corrupted
# (regression or bug-fix compared to Python 3.4)
# - read1(), which fetches data before raising EOFError on next call
# works here but is only available from Python>=3.3
# - scrapy does not support Python 3.2
# - Python 2.7 GzipFile works fine with standard read() + extrabuf
if six.PY3:
def read1(gzf, size=-1):
return gzf.read1(size)
else:
def read1(gzf, size=-1):
return gzf.read(size)
def gunzip(data):
"""Gunzip the given data and return as much data as possible.
This is resilient to CRC checksum errors.
"""
f = ReadOneGzipFile(fileobj=BytesIO(data))
f = GzipFile(fileobj=BytesIO(data))
output = b''
chunk = b'.'
while chunk:
try:
chunk = f.readone(8196)
chunk = read1(f, 8196)
output += chunk
except (IOError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise