diff --git a/.travis.yml b/.travis.yml index 906115096..4f44d1e6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,19 +11,16 @@ matrix: env: TOXENV=py27 - python: 2.7 env: TOXENV=jessie + - python: 2.7 + env: TOXENV=pypy - python: 3.3 env: TOXENV=py33 - python: 3.5 env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 2.7 - env: TOXENV=pypy - python: 3.6 env: TOXENV=docs - allow_failures: - - python: 2.7 - env: TOXENV=pypy install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e4dc7f2de..dc6b59fe0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings +from scrapy.utils.python import garbage_collect from scrapy.settings.deprecated import check_deprecated_settings def _iter_command_classes(module_name): @@ -165,4 +166,9 @@ def _run_command_profiled(cmd, args, opts): p.dump_stats(opts.profile) if __name__ == '__main__': - execute() + try: + execute() + finally: + # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() + # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies + garbage_collect() diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 2fb4b6a15..648b32ec7 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -13,7 +13,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes, to_unicode, garbage_collect logger = logging.getLogger(__name__) @@ -362,6 +362,7 @@ class LeveldbCacheStorage(object): # avoid them being removed in storages with timestamp-based autoremoval. self.db.CompactRange() del self.db + garbage_collect() def retrieve_response(self, spider, request): data = self._read_data(spider, request) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 4c500abf4..72f8f4311 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import gc import os import re import inspect @@ -8,6 +9,8 @@ import weakref import errno import six from functools import partial, wraps +import sys +import time from scrapy.utils.decorators import deprecated @@ -355,3 +358,13 @@ def global_object_name(obj): 'scrapy.http.request.Request' """ return "%s.%s" % (obj.__module__, obj.__name__) + + +if hasattr(sys, "pypy_version_info"): + def garbage_collect(): + # Collecting weakreferences can take two collections on PyPy. + gc.collect() + gc.collect() +else: + def garbage_collect(): + gc.collect() diff --git a/setup.py b/setup.py index 086ab8142..c03f0b9f7 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,31 @@ from os.path import dirname, join -from setuptools import setup, find_packages +from pkg_resources import parse_version +from setuptools import setup, find_packages, __version__ as setuptools_version with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() +def has_environment_marker_platform_impl_support(): + """Code extracted from 'pytest/setup.py' + https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 + + The first known release to support environment marker with range operators + it is 18.5, see: + https://setuptools.readthedocs.io/en/latest/history.html#id235 + """ + return parse_version(setuptools_version) >= parse_version('18.5') + + +extras_require = {} + +if has_environment_marker_platform_impl_support(): + extras_require[':platform_python_implementation == "PyPy"'] = [ + 'PyPyDispatcher>=2.1.0', + ] + + setup( name='Scrapy', version=version, @@ -53,4 +73,5 @@ setup( 'PyDispatcher>=2.0.5', 'service_identity', ], + extras_require=extras_require, ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 2693a18d9..9d07eb95b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -290,7 +290,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() il.add_value('name', [u'$10']) try: - float('$10') + float(u'$10') except Exception as e: expected_exc_str = str(e) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..3a24348b4 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,3 +1,5 @@ +import pickle + from queuelib.tests import test_queue as t from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue from scrapy.item import Item, Field @@ -14,6 +16,22 @@ class TestLoader(ItemLoader): default_item_class = TestItem name_out = staticmethod(_test_procesor) +def nonserializable_object_test(self): + try: + pickle.dumps(lambda x: x) + except Exception: + # Trigger Twisted bug #7989 + import twisted.persisted.styles # NOQA + q = self.queue() + self.assertRaises(ValueError, q.push, lambda x: x) + else: + # Use a different unpickleable object + class A(object): pass + a = A() + a.__reduce__ = a.__reduce_ex__ = None + q = self.queue() + self.assertRaises(ValueError, q.push, a) + class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): chunksize = 100000 @@ -30,11 +48,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): self.assertEqual(q.pop(), 123) self.assertEqual(q.pop(), {'a': 'dict'}) - def test_nonserializable_object(self): - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + test_nonserializable_object = nonserializable_object_test class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -110,11 +124,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): self.assertEqual(q.pop(), 123) self.assertEqual(q.pop(), 'a') - def test_nonserializable_object(self): - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + test_nonserializable_object = nonserializable_object_test class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 9a0cc975d..8becca0f1 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,7 +1,9 @@ +import gc import functools import operator import unittest from itertools import count +import platform import six from scrapy.utils.python import ( @@ -144,6 +146,9 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertNotEqual(v, wk[_Weakme()]) self.assertEqual(v, wk[k]) del k + for _ in range(100): + if wk._weakdict: + gc.collect() self.assertFalse(len(wk._weakdict)) @unittest.skipUnless(six.PY2, "deprecated function") @@ -208,10 +213,16 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(cal), ['a', 'b', 'c']) self.assertEqual(get_func_args(object), []) - # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(six.text_type.split), []) - self.assertEqual(get_func_args(" ".join), []) - self.assertEqual(get_func_args(operator.itemgetter(2)), []) + if platform.python_implementation() == 'CPython': + # TODO: how do we fix this to return the actual argument names? + self.assertEqual(get_func_args(six.text_type.split), []) + self.assertEqual(get_func_args(" ".join), []) + self.assertEqual(get_func_args(operator.itemgetter(2)), []) + else: + self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join), ['list']) + self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj']) + def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])