From a2854e39481ffb79cdca71e28ab03fec2288a25b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 25 Nov 2009 22:28:59 -0200 Subject: [PATCH] Added hack to speed up processing of IgnoreRequest errors (#125) --- scrapy/utils/defer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 78ef7016a..bf9014bbd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,6 +5,8 @@ Helper functions for dealing with Twisted deferreds from twisted.internet import defer, reactor, task from twisted.python import failure +from scrapy.core.exceptions import IgnoreRequest + def defer_fail(_failure): """Same as twisted.internet.defer.fail, but delay calling errback until next reactor loop @@ -35,6 +37,11 @@ def mustbe_deferred(f, *args, **kw): """ try: result = f(*args, **kw) + # FIXME: Hack to avoid introspecting tracebacks. This to speed up + # processing of IgnoreRequest errors which are, by far, the most common + # exception in Scrapy - see #125 + except IgnoreRequest, e: + return defer_fail(failure.Failure(e)) except: return defer_fail(failure.Failure()) else: