Merge pull request #1253 from scrapy/deferdelay

Increase delay of defer calls

docs/news.rst

@@ -256,6 +256,7 @@ New Features and Enhancements
 - Add project name validation (:issue:`817`)
 - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`,
   :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`)
+- Be more responsive with IO operations (:issue:`1074` and :issue:`1075`)
 
 Deprecations and Removals
 

scrapy/utils/defer.py

@@ -8,19 +8,25 @@ from twisted.python import failure
 from scrapy.exceptions import IgnoreRequest
 
 def defer_fail(_failure):
-    """Same as twisted.internet.defer.fail, but delay calling errback until
+    """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
+
+    It delays by 100ms so reactor has a chance to go trough readers and writers
+    before attending pending delayed calls, so do not set delay to zero.
     """
     d = defer.Deferred()
-    reactor.callLater(0, d.errback, _failure)
+    reactor.callLater(0.1, d.errback, _failure)
     return d
 
 def defer_succeed(result):
-    """Same as twsited.internet.defer.succed, but delay calling callback until
+    """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
+
+    It delays by 100ms so reactor has a chance to go trough readers and writers
+    before attending pending delayed calls, so do not set delay to zero.
     """
     d = defer.Deferred()
-    reactor.callLater(0, d.callback, result)
+    reactor.callLater(0.1, d.callback, result)
     return d
 
 def defer_result(result):