From c74903f9da8d85909c7225c6a107a109f98056d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sat, 9 Nov 2013 02:12:52 -0200 Subject: [PATCH] process_parallel was leaking the failures on its internal deferreds. closes #458 DeferredList implemented cancellation in Twisted 13.2.0 by holding a reference to the affected deferreds objects, if a deferred errored the result was propagated to the DeferredList but still referenced by the original deferred and nobody was consuming it. The tests started to fail because the reference from DeferredList prevented the underlining deferred from been collected before the test finish invalidating the effect of self.flushedLoggedErrors() call. --- scrapy/utils/defer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 90fb4d9d4..45a451d08 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -82,8 +82,8 @@ def process_parallel(callbacks, input, *a, **kw): callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.gatherResults(dfds) - d.addErrback(lambda _: _.value.subFailure) + d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1) + d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d def iter_errback(iterable, errback, *a, **kw):