diff --git a/.travis.yml b/.travis.yml index e857abbd8..ae9c745ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 2.7 +python: 3.5 sudo: false branches: only: @@ -9,6 +9,7 @@ env: - TOXENV=py27 - TOXENV=precise - TOXENV=py33 + - TOXENV=py35 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6e2a60fd2..21520f454 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,7 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - except pickle.PicklingError as e: + # Python>=3.5 raises AttributeError here while + # Python<=3.4 raises pickle.PicklingError + except (pickle.PicklingError, AttributeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fa4bba57..d035f9fdf 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,9 +4,25 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - from gzip import GzipFile +import six + + +# - Python>=3.5 GzipFile's read() has issues returning leftover +# uncompressed data when input is corrupted +# (regression or bug-fix compared to Python 3.4) +# - read1(), which fetches data before raising EOFError on next call +# works here but is only available from Python>=3.3 +# - scrapy does not support Python 3.2 +# - Python 2.7 GzipFile works fine with standard read() + extrabuf +if six.PY2: + def read1(gzf, size=-1): + return gzf.read(size) +else: + def read1(gzf, size=-1): + return gzf.read1(size) + def gunzip(data): """Gunzip the given data and return as much data as possible. @@ -18,16 +34,18 @@ def gunzip(data): chunk = b'.' while chunk: try: - chunk = f.read(8196) + chunk = read1(f, 8196) output += chunk except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf # contains the whole page content - if output or f.extrabuf: - output += f.extrabuf - break + if output or getattr(f, 'extrabuf', None): + try: + output += f.extrabuf + finally: + break else: raise return output diff --git a/tox.ini b/tox.ini index eae7e8e47..b8d45d5b9 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,10 @@ deps = basepython = python3.4 deps = {[testenv:py33]deps} +[testenv:py35] +basepython = python3.5 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps =