From 1f2233837a4219d02be73dc0836dfc885d47fffb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 16:58:24 +0100 Subject: [PATCH] Use if Py2/Py3 function instead of custom GzipFile class method --- scrapy/utils/gz.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index df1d29698..3e6596b0b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,24 +7,35 @@ except ImportError: from io import UnsupportedOperation from gzip import GzipFile -class ReadOneGzipFile(GzipFile): - def readone(self, size=-1): - try: - return self.read1(size) - except UnsupportedOperation: - return self.read(size) +import six + + +# - Python>=3.5 GzipFile's read() has issues returning leftover +# uncompressed data when input is corrupted +# (regression or bug-fix compared to Python 3.4) +# - read1(), which fetches data before raising EOFError on next call +# works here but is only available from Python>=3.3 +# - scrapy does not support Python 3.2 +# - Python 2.7 GzipFile works fine with standard read() + extrabuf +if six.PY3: + def read1(gzf, size=-1): + return gzf.read1(size) +else: + def read1(gzf, size=-1): + return gzf.read(size) + def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = ReadOneGzipFile(fileobj=BytesIO(data)) + f = GzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.readone(8196) + chunk = read1(f, 8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise