Move garbage_collect to scrapy.utils.python

scrapy/cmdline.py

@@ -4,7 +4,6 @@ import optparse
 import cProfile
 import inspect
 import pkg_resources
-import gc
 
 import scrapy
 from scrapy.crawler import CrawlerProcess
@@ -12,6 +11,7 @@ from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.utils.python import garbage_collect
 from scrapy.settings.deprecated import check_deprecated_settings
 
 def _iter_command_classes(module_name):
@@ -171,4 +171,4 @@ if __name__ == '__main__':
     finally:
         # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
         # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
-        gc.collect()
+        garbage_collect()

scrapy/utils/python.py

@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+import gc
 import os
 import re
 import inspect
@@ -8,6 +9,8 @@ import weakref
 import errno
 import six
 from functools import partial, wraps
+import sys
+import time
 
 from scrapy.utils.decorators import deprecated
 
@@ -355,3 +358,20 @@ def global_object_name(obj):
     'scrapy.http.request.Request'
     """
     return "%s.%s" % (obj.__module__, obj.__name__)
+
+
+if sys.platform.startswith('java'):
+    def garbage_collect():
+        # Some JVM GCs will execute finalizers in a different thread, meaning
+        # we need to wait for that to complete before we go on looking for the
+        # effects of that.
+        gc.collect()
+        time.sleep(0.1)
+elif hasattr(sys, "pypy_version_info"):
+    def garbage_collect():
+        # Collecting weakreferences can take two collections on PyPy.
+        gc.collect()
+        gc.collect()
+else:
+    def garbage_collect():
+        gc.collect()