From a93d49a64ca170d98de98ee44a181ced04a23bea Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:47:42 +0100 Subject: [PATCH 1/9] Add Python 3.5 tox env and Python 3.4 tests in Travis CI --- .travis.yml | 1 + tox.ini | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index e857abbd8..65cfaad03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 + - TOXENV=py34 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index eae7e8e47..b8d45d5b9 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,10 @@ deps = basepython = python3.4 deps = {[testenv:py33]deps} +[testenv:py35] +basepython = python3.5 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps = From 85b0e6c9c766218bd48268f0c115c77b3886e539 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 14 Jan 2016 10:50:51 +0100 Subject: [PATCH 2/9] Travis: run tox with Python 3.5 + add Python 3.5 tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65cfaad03..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 2.7 +python: 3.5 sudo: false branches: only: @@ -9,7 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 - - TOXENV=py34 + - TOXENV=py35 - TOXENV=docs install: - pip install -U tox twine wheel codecov From 131f4632472922234a7abde116eb566329a952a2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 11:17:52 +0100 Subject: [PATCH 3/9] Allow failures for Python 3.5 for now --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ae9c745ac..ac93e337d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs +matrix: + allow_failures: + - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox From ee4fadc00724f02f9098625ac4d72fb29eac4dcf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 14:57:15 +0100 Subject: [PATCH 4/9] Use .read1() if available when using GzipFile --- .travis.yml | 3 --- scrapy/utils/gz.py | 20 ++++++++++++++------ tests/test_squeues.py | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac93e337d..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,6 @@ env: - TOXENV=py33 - TOXENV=py35 - TOXENV=docs -matrix: - allow_failures: - - env: TOXENV=py35 install: - pip install -U tox twine wheel codecov script: tox diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fa4bba57..df1d29698 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,30 +4,38 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import UnsupportedOperation from gzip import GzipFile +class ReadOneGzipFile(GzipFile): + def readone(self, size=-1): + try: + return self.read1(size) + except UnsupportedOperation: + return self.read(size) def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = GzipFile(fileobj=BytesIO(data)) + f = ReadOneGzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.read(8196) + chunk = f.readone(8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf # contains the whole page content - if output or f.extrabuf: - output += f.extrabuf - break + if output or getattr(f, 'extrabuf', None): + try: + output += f.extrabuf + finally: + break else: raise return output diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..232f539e6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From bcbad2905d3ff375e85ede723d13560d765ee729 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Jan 2016 15:22:29 +0100 Subject: [PATCH 5/9] Stick with ValueError for queue/serialization exception tests --- tests/test_squeues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 232f539e6..48871ceeb 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -34,7 +34,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) + self.assertRaises(ValueError, q.push, lambda x: x) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -114,7 +114,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA q = self.queue() - self.assertRaises((ValueError, AttributeError), q.push, lambda x: x) + self.assertRaises(ValueError, q.push, lambda x: x) class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From fd99ef86dfca50dbd36b2c1a022cf30a0720dbea Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Jan 2016 17:57:55 +0100 Subject: [PATCH 6/9] Test for AttributeError when pickling objects (Python>=3.5) Same "fix" as in e.g. https://github.com/joblib/joblib/pull/246 --- scrapy/squeues.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6e2a60fd2..21520f454 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,7 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - except pickle.PicklingError as e: + # Python>=3.5 raises AttributeError here while + # Python<=3.4 raises pickle.PicklingError + except (pickle.PicklingError, AttributeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ From 1f2233837a4219d02be73dc0836dfc885d47fffb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 16:58:24 +0100 Subject: [PATCH 7/9] Use if Py2/Py3 function instead of custom GzipFile class method --- scrapy/utils/gz.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index df1d29698..3e6596b0b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,24 +7,35 @@ except ImportError: from io import UnsupportedOperation from gzip import GzipFile -class ReadOneGzipFile(GzipFile): - def readone(self, size=-1): - try: - return self.read1(size) - except UnsupportedOperation: - return self.read(size) +import six + + +# - Python>=3.5 GzipFile's read() has issues returning leftover +# uncompressed data when input is corrupted +# (regression or bug-fix compared to Python 3.4) +# - read1(), which fetches data before raising EOFError on next call +# works here but is only available from Python>=3.3 +# - scrapy does not support Python 3.2 +# - Python 2.7 GzipFile works fine with standard read() + extrabuf +if six.PY3: + def read1(gzf, size=-1): + return gzf.read1(size) +else: + def read1(gzf, size=-1): + return gzf.read(size) + def gunzip(data): """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ - f = ReadOneGzipFile(fileobj=BytesIO(data)) + f = GzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: try: - chunk = f.readone(8196) + chunk = read1(f, 8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise From 2b5245839ceaaa256c7edccee351ea79129fa3e4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Jan 2016 17:04:57 +0100 Subject: [PATCH 8/9] Remove unused import statement --- scrapy/utils/gz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 3e6596b0b..d69fb598d 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,7 +4,6 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO -from io import UnsupportedOperation from gzip import GzipFile import six From 29ff84a7920dd27a7b723f21b68ccc6e2076c08a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 20 Jan 2016 12:03:38 +0100 Subject: [PATCH 9/9] Invert PY2/PY3 test for conditional read1() definition --- scrapy/utils/gz.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index d69fb598d..d035f9fdf 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -16,12 +16,12 @@ import six # works here but is only available from Python>=3.3 # - scrapy does not support Python 3.2 # - Python 2.7 GzipFile works fine with standard read() + extrabuf -if six.PY3: - def read1(gzf, size=-1): - return gzf.read1(size) -else: +if six.PY2: def read1(gzf, size=-1): return gzf.read(size) +else: + def read1(gzf, size=-1): + return gzf.read1(size) def gunzip(data):