diff --git a/docs/news.rst b/docs/news.rst index 8c093dadc..ebc5304d1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -256,6 +256,7 @@ New Features and Enhancements - Add project name validation (:issue:`817`) - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`, :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`) +- Be more responsive with IO operations (:issue:`1074` and :issue:`1075`) Deprecations and Removals diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 45a451d08..97c41af5e 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,19 +8,25 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest def defer_fail(_failure): - """Same as twisted.internet.defer.fail, but delay calling errback until + """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop + + It delays by 100ms so reactor has a chance to go trough readers and writers + before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() - reactor.callLater(0, d.errback, _failure) + reactor.callLater(0.1, d.errback, _failure) return d def defer_succeed(result): - """Same as twsited.internet.defer.succed, but delay calling callback until + """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop + + It delays by 100ms so reactor has a chance to go trough readers and writers + before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() - reactor.callLater(0, d.callback, result) + reactor.callLater(0.1, d.callback, result) return d def defer_result(result):