Merge pull request #2793 from lopuhin/pypy2

Fix PyPy test failures

.travis.yml

@@ -11,19 +11,16 @@ matrix:
       env: TOXENV=py27
     - python: 2.7
       env: TOXENV=jessie
+    - python: 2.7
+      env: TOXENV=pypy
     - python: 3.3
       env: TOXENV=py33
     - python: 3.5
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
-    - python: 2.7
-      env: TOXENV=pypy
     - python: 3.6
       env: TOXENV=docs
-  allow_failures:
-    - python: 2.7
-      env: TOXENV=pypy
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then

scrapy/cmdline.py

@@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.utils.python import garbage_collect
 from scrapy.settings.deprecated import check_deprecated_settings
 
 def _iter_command_classes(module_name):
@@ -165,4 +166,9 @@ def _run_command_profiled(cmd, args, opts):
         p.dump_stats(opts.profile)
 
 if __name__ == '__main__':
-    execute()
+    try:
+        execute()
+    finally:
+        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
+        # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
+        garbage_collect()

scrapy/extensions/httpcache.py

@@ -13,7 +13,7 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.project import data_path
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
 
 
 logger = logging.getLogger(__name__)
@@ -362,6 +362,7 @@ class LeveldbCacheStorage(object):
         # avoid them being removed in storages with timestamp-based autoremoval.
         self.db.CompactRange()
         del self.db
+        garbage_collect()
 
     def retrieve_response(self, spider, request):
         data = self._read_data(spider, request)

scrapy/utils/python.py

@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+import gc
 import os
 import re
 import inspect
@@ -8,6 +9,8 @@ import weakref
 import errno
 import six
 from functools import partial, wraps
+import sys
+import time
 
 from scrapy.utils.decorators import deprecated
 
@@ -355,3 +358,13 @@ def global_object_name(obj):
     'scrapy.http.request.Request'
     """
     return "%s.%s" % (obj.__module__, obj.__name__)
+
+
+if hasattr(sys, "pypy_version_info"):
+    def garbage_collect():
+        # Collecting weakreferences can take two collections on PyPy.
+        gc.collect()
+        gc.collect()
+else:
+    def garbage_collect():
+        gc.collect()

setup.py

@@ -1,11 +1,31 @@
 from os.path import dirname, join
-from setuptools import setup, find_packages
+from pkg_resources import parse_version
+from setuptools import setup, find_packages, __version__ as setuptools_version
 
 
 with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
     version = f.read().decode('ascii').strip()
 
 
+def has_environment_marker_platform_impl_support():
+    """Code extracted from 'pytest/setup.py'
+    https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31
+
+    The first known release to support environment marker with range operators
+    it is 18.5, see:
+    https://setuptools.readthedocs.io/en/latest/history.html#id235
+    """
+    return parse_version(setuptools_version) >= parse_version('18.5')
+
+
+extras_require = {}
+
+if has_environment_marker_platform_impl_support():
+    extras_require[':platform_python_implementation == "PyPy"'] = [
+        'PyPyDispatcher>=2.1.0',
+    ]
+
+
 setup(
     name='Scrapy',
     version=version,
@@ -53,4 +73,5 @@ setup(
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],
+    extras_require=extras_require,
 )

tests/test_loader.py

@@ -290,7 +290,7 @@ class BasicItemLoaderTest(unittest.TestCase):
         il = TestItemLoader()
         il.add_value('name', [u'$10'])
         try:
-            float('$10')
+            float(u'$10')
         except Exception as e:
             expected_exc_str = str(e)
 

tests/test_squeues.py

@@ -1,3 +1,5 @@
+import pickle
+
 from queuelib.tests import test_queue as t
 from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
 from scrapy.item import Item, Field
@@ -14,6 +16,22 @@ class TestLoader(ItemLoader):
     default_item_class = TestItem
     name_out = staticmethod(_test_procesor)
 
+def nonserializable_object_test(self):
+    try:
+        pickle.dumps(lambda x: x)
+    except Exception:
+        # Trigger Twisted bug #7989
+        import twisted.persisted.styles  # NOQA
+        q = self.queue()
+        self.assertRaises(ValueError, q.push, lambda x: x)
+    else:
+        # Use a different unpickleable object
+        class A(object): pass
+        a = A()
+        a.__reduce__ = a.__reduce_ex__ = None
+        q = self.queue()
+        self.assertRaises(ValueError, q.push, a)
+
 class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
 
     chunksize = 100000
@@ -30,11 +48,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
         self.assertEqual(q.pop(), 123)
         self.assertEqual(q.pop(), {'a': 'dict'})
 
-    def test_nonserializable_object(self):
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        q = self.queue()
-        self.assertRaises(ValueError, q.push, lambda x: x)
+    test_nonserializable_object = nonserializable_object_test
 
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
@@ -110,11 +124,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
         self.assertEqual(q.pop(), 123)
         self.assertEqual(q.pop(), 'a')
 
-    def test_nonserializable_object(self):
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        q = self.queue()
-        self.assertRaises(ValueError, q.push, lambda x: x)
+    test_nonserializable_object = nonserializable_object_test
 
 
 class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):

tests/test_utils_python.py

@@ -1,7 +1,9 @@
+import gc
 import functools
 import operator
 import unittest
 from itertools import count
+import platform
 import six
 
 from scrapy.utils.python import (
@@ -144,6 +146,9 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertNotEqual(v, wk[_Weakme()])
         self.assertEqual(v, wk[k])
         del k
+        for _ in range(100):
+            if wk._weakdict:
+                gc.collect()
         self.assertFalse(len(wk._weakdict))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
@@ -208,10 +213,16 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(object), [])
 
-        # TODO: how do we fix this to return the actual argument names?
-        self.assertEqual(get_func_args(six.text_type.split), [])
-        self.assertEqual(get_func_args(" ".join), [])
-        self.assertEqual(get_func_args(operator.itemgetter(2)), [])
+        if platform.python_implementation() == 'CPython':
+            # TODO: how do we fix this to return the actual argument names?
+            self.assertEqual(get_func_args(six.text_type.split), [])
+            self.assertEqual(get_func_args(" ".join), [])
+            self.assertEqual(get_func_args(operator.itemgetter(2)), [])
+        else:
+            self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join), ['list'])
+            self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj'])
+
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])