diff --git a/.travis.yml b/.travis.yml index ac93e337d..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,6 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs -matrix: - allow_failures: - - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fa4bba57..df1d29698 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,30 +4,38 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import UnsupportedOperation from gzip import GzipFile +class ReadOneGzipFile(GzipFile): + def readone(self, size=-1): + try: + return self.read1(size) + except UnsupportedOperation: + return self.read(size) def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = GzipFile(fileobj=BytesIO(data)) + f = ReadOneGzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.read(8196) + chunk = f.readone(8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf # contains the whole page content - if output or f.extrabuf: - output += f.extrabuf - break + if output or getattr(f, 'extrabuf', None): + try: + output += f.extrabuf + finally: + break else: raise return output diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..232f539e6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):