Merge pull request #1674 from redapple/tox-py35

[MRG+1] Add Python 3.5 tox env + Python 3.5 tests in Travis

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 2.7
+python: 3.5
 sudo: false
 branches:
   only:
@@ -9,6 +9,7 @@ env:
  - TOXENV=py27
  - TOXENV=precise
  - TOXENV=py33
+ - TOXENV=py35
  - TOXENV=docs
 install:
  - pip install -U tox twine wheel codecov

scrapy/squeues.py

@@ -25,7 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
-    except pickle.PicklingError as e:
+    # Python>=3.5 raises AttributeError here while
+    # Python<=3.4 raises pickle.PicklingError
+    except (pickle.PicklingError, AttributeError) as e:
         raise ValueError(str(e))
 
 PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \

scrapy/utils/gz.py

@@ -4,9 +4,25 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-
 from gzip import GzipFile
 
+import six
+
+
+# - Python>=3.5 GzipFile's read() has issues returning leftover
+#   uncompressed data when input is corrupted
+#   (regression or bug-fix compared to Python 3.4)
+# - read1(), which fetches data before raising EOFError on next call
+#   works here but is only available from Python>=3.3
+# - scrapy does not support Python 3.2
+# - Python 2.7 GzipFile works fine with standard read() + extrabuf
+if six.PY2:
+    def read1(gzf, size=-1):
+        return gzf.read(size)
+else:
+    def read1(gzf, size=-1):
+        return gzf.read1(size)
+
 
 def gunzip(data):
     """Gunzip the given data and return as much data as possible.
@@ -18,16 +34,18 @@ def gunzip(data):
     chunk = b'.'
     while chunk:
         try:
-            chunk = f.read(8196)
+            chunk = read1(f, 8196)
             output += chunk
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
             # some pages are quite small so output is '' and f.extrabuf
             # contains the whole page content
-            if output or f.extrabuf:
-                output += f.extrabuf
-                break
+            if output or getattr(f, 'extrabuf', None):
+                try:
+                    output += f.extrabuf
+                finally:
+                    break
             else:
                 raise
     return output

tox.ini

@@ -48,6 +48,10 @@ deps =
 basepython = python3.4
 deps = {[testenv:py33]deps}
 
+[testenv:py35]
+basepython = python3.5
+deps = {[testenv:py33]deps}
+
 [docs]
 changedir = docs
 deps =