From 419a78015958f3b7bc2506881a21559d6231aa40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 9 Nov 2013 02:12:52 -0200 Subject: [PATCH] process_parallel was leaking the failures on its internal deferreds. closes #458 DeferredList implemented cancellation in Twisted 13.2.0 by holding a reference to the affected deferreds objects, if a deferred errored the result was propagated to the DeferredList but still referenced by the original deferred and nobody was consuming it. The tests started to fail because the reference from DeferredList prevented the underlining deferred from been collected before the test finish invalidating the effect of self.flushedLoggedErrors() call. --- scrapy/utils/defer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 90fb4d9d4..45a451d08 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -82,8 +82,8 @@ def process_parallel(callbacks, input, *a, **kw): callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.gatherResults(dfds) - d.addErrback(lambda _: _.value.subFailure) + d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1) + d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d def iter_errback(iterable, errback, *a, **kw):