From 9b78f929def2f9b40353d832ecc197e0faf6da4a Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 23 Jun 2009 14:45:16 -0700 Subject: [PATCH] remove obsolete deferred_imap util, use coiterate+imap instead --- scrapy/tests/test_utils_defer.py | 37 -------------------------------- scrapy/utils/defer.py | 36 ------------------------------- 2 files changed, 73 deletions(-) delete mode 100644 scrapy/tests/test_utils_defer.py diff --git a/scrapy/tests/test_utils_defer.py b/scrapy/tests/test_utils_defer.py deleted file mode 100644 index 2248c0032..000000000 --- a/scrapy/tests/test_utils_defer.py +++ /dev/null @@ -1,37 +0,0 @@ -from itertools import imap -from twisted.trial import unittest - -from scrapy.utils.defer import deferred_imap - - -class DeferTest(unittest.TestCase): - def test_deferred_imap_1(self): - """deferred_imap storing results""" - seq = [1, 2, 3] - output = list(imap(None, seq)) - - dfd = deferred_imap(None, seq) - dfd.addCallback(self.assertEqual, output) - return dfd - - def test_deferred_imap_2(self): - """deferred_imap not storing results""" - seq = [1, 2, 3] - output = list(imap(None, seq, seq)) - - dfd = deferred_imap(None, seq, seq) - dfd.addCallback(self.assertEqual, output) - return dfd - - def test_deferred_imap_3(self): - """deferred_imap not storing results""" - seq = [1, 2, 3] - output = [] - function = lambda v: output.append(v) - - dfd = deferred_imap(function, seq, store_results=False) - dfd.addCallback(self.assertEqual, []) - dfd.addCallback(lambda _: self.assertEqual(output, seq)) - return dfd - - diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 5b46d1b33..de24eecc1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -2,7 +2,6 @@ Helper functions for dealing with Twisted deferreds """ -from itertools import imap from twisted.internet import defer, reactor from twisted.python import failure @@ -38,38 +37,3 @@ def mustbe_deferred(f, *args, **kw): def chain_deferred(d1, d2): return d1.chainDeferred(d2).addBoth(lambda _:d2) -def deferred_imap(function, *sequences, **kwargs): - """Analog to itertools.imap python function but friendly iterable evaluation - taking in count cooperative multitasking. - - It returns a Deferred object that is fired when StopIteration is reached or - when any exception is raised when calling function. - - By default the output of the evaluation is collected into a list and - returned as deferred result when iterable finished. But it can be disabled - (to save memory) using `store_results` parameter. - - """ - - next_delay = kwargs.pop('next_delay', 0) - store_results = kwargs.pop('store_results', True) - - deferred = defer.Deferred() - container = [] - - iterator = imap(function, *sequences) - - def _next(): - try: - value = iterator.next() - if store_results: - container.append(value) - except StopIteration: - reactor.callLater(0, deferred.callback, container) - except: - reactor.callLater(0, deferred.errback, failure.Failure()) - else: - reactor.callLater(next_delay, _next) - - _next() - return deferred