From 27ce3225bdf843567e037c2cdba387d0b75e24ef Mon Sep 17 00:00:00 2001 From: Alexey Vishnevsky Date: Mon, 16 Mar 2015 11:00:27 +0200 Subject: [PATCH 1/2] Makes scrapy more async by letting to reactor spend another couple of cycles to accomplish its needs. --- scrapy/utils/defer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 45a451d08..5df93748d 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -12,7 +12,7 @@ def defer_fail(_failure): next reactor loop """ d = defer.Deferred() - reactor.callLater(0, d.errback, _failure) + reactor.callLater(0.1, d.errback, _failure) return d def defer_succeed(result): @@ -20,7 +20,7 @@ def defer_succeed(result): next reactor loop """ d = defer.Deferred() - reactor.callLater(0, d.callback, result) + reactor.callLater(0.1, d.callback, result) return d def defer_result(result): From d439c26d76468d1732128f037a646f72409ea135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 22 May 2015 17:12:43 -0300 Subject: [PATCH 2/2] update docstring and release notes --- docs/news.rst | 1 + scrapy/utils/defer.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 8c093dadc..ebc5304d1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -256,6 +256,7 @@ New Features and Enhancements - Add project name validation (:issue:`817`) - GSoC API cleanup (:issue:`816`, :issue:`1128`, :issue:`1147`, :issue:`1148`, :issue:`1156`, :issue:`1185`, :issue:`1187`) +- Be more responsive with IO operations (:issue:`1074` and :issue:`1075`) Deprecations and Removals diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 5df93748d..97c41af5e 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,16 +8,22 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest def defer_fail(_failure): - """Same as twisted.internet.defer.fail, but delay calling errback until + """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop + + It delays by 100ms so reactor has a chance to go trough readers and writers + before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() reactor.callLater(0.1, d.errback, _failure) return d def defer_succeed(result): - """Same as twsited.internet.defer.succed, but delay calling callback until + """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop + + It delays by 100ms so reactor has a chance to go trough readers and writers + before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() reactor.callLater(0.1, d.callback, result)