From cb8e0d9bdd7524d8dfd1fe5122701641bebc2ffa Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 23 Sep 2008 21:48:48 +0000 Subject: [PATCH] added scrapy.utils.defer, moved deferred functions from scrapy.utils.misc to that module --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40271 --- scrapy/trunk/scrapy/contrib/pipeline/media.py | 2 +- scrapy/trunk/scrapy/contrib/web/json.py | 2 +- scrapy/trunk/scrapy/contrib/web/site.py | 2 +- .../trunk/scrapy/core/downloader/handlers.py | 2 +- .../trunk/scrapy/core/downloader/manager.py | 3 +- .../scrapy/core/downloader/middleware.py | 3 +- scrapy/trunk/scrapy/core/engine.py | 2 +- scrapy/trunk/scrapy/core/manager.py | 2 +- .../trunk/scrapy/core/scheduler/schedulers.py | 2 +- scrapy/trunk/scrapy/http/request.py | 2 +- scrapy/trunk/scrapy/item/pipeline.py | 3 +- scrapy/trunk/scrapy/spider/middleware.py | 3 +- scrapy/trunk/scrapy/utils/defer.py | 81 ++++++++++++++++++ scrapy/trunk/scrapy/utils/misc.py | 85 +------------------ 14 files changed, 98 insertions(+), 96 deletions(-) create mode 100644 scrapy/trunk/scrapy/utils/defer.py diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/trunk/scrapy/contrib/pipeline/media.py index b19519e6e..266f984c0 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/media.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/media.py @@ -1,6 +1,6 @@ from twisted.internet import defer -from scrapy.utils.misc import mustbe_deferred, defer_result +from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.core import log from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import DropItem, NotConfigured diff --git a/scrapy/trunk/scrapy/contrib/web/json.py b/scrapy/trunk/scrapy/contrib/web/json.py index 6c7aa06fa..2f0100836 100644 --- a/scrapy/trunk/scrapy/contrib/web/json.py +++ b/scrapy/trunk/scrapy/contrib/web/json.py @@ -1,6 +1,6 @@ from functools import wraps from twisted.internet import defer -from scrapy.utils.misc import mustbe_deferred +from scrapy.utils.defer import mustbe_deferred from .http import HttpResponse diff --git a/scrapy/trunk/scrapy/contrib/web/site.py b/scrapy/trunk/scrapy/contrib/web/site.py index 5ba47ff1c..40c23772b 100644 --- a/scrapy/trunk/scrapy/contrib/web/site.py +++ b/scrapy/trunk/scrapy/contrib/web/site.py @@ -61,7 +61,7 @@ import re from twisted.web import server, resource from twisted.internet import reactor -from scrapy.utils.misc import mustbe_deferred +from scrapy.utils.defer import mustbe_deferred from .http import HttpResponse, build_httprequest diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py index bbcd69aa4..e63e501c4 100644 --- a/scrapy/trunk/scrapy/core/downloader/handlers.py +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -13,7 +13,7 @@ from twisted.web import error as web_error from scrapy.core import signals from scrapy.http import Request, Response, Headers from scrapy.core.exceptions import UsageError, HttpException -from scrapy.utils.misc import defer_succeed +from scrapy.utils.defer import defer_succeed from scrapy.conf import settings def download_any(request, spider): diff --git a/scrapy/trunk/scrapy/core/downloader/manager.py b/scrapy/trunk/scrapy/core/downloader/manager.py index b3b367b7a..9ef73e42e 100644 --- a/scrapy/trunk/scrapy/core/downloader/manager.py +++ b/scrapy/trunk/scrapy/core/downloader/manager.py @@ -12,7 +12,7 @@ from scrapy.core.downloader.handlers import download_any from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core import log from scrapy.conf import settings -from scrapy.utils.misc import chain_deferred +from scrapy.utils.defer import chain_deferred, mustbe_deferred class SiteDetails(object): @@ -126,7 +126,6 @@ class Downloader(object): def _finish(result): self.process_queue(spider) - from scrapy.utils.misc import mustbe_deferred dwld = mustbe_deferred(self.download_function, request, spider) dwld.addBoth(_remove) chain_deferred(dwld, deferred) diff --git a/scrapy/trunk/scrapy/core/downloader/middleware.py b/scrapy/trunk/scrapy/core/downloader/middleware.py index bf01017a0..a6f7088d5 100644 --- a/scrapy/trunk/scrapy/core/downloader/middleware.py +++ b/scrapy/trunk/scrapy/core/downloader/middleware.py @@ -4,7 +4,8 @@ request-response middleware extension from scrapy.core import signals, log from scrapy.http import Request, Response from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class, mustbe_deferred +from scrapy.utils.misc import load_class +from scrapy.utils.defer import mustbe_deferred from scrapy.core.downloader.handlers import download_any from scrapy.conf import settings diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py index 055ac335b..1c83f1dc0 100644 --- a/scrapy/trunk/scrapy/core/engine.py +++ b/scrapy/trunk/scrapy/core/engine.py @@ -17,7 +17,7 @@ from scrapy.item import ScrapedItem from scrapy.item.pipeline import ItemPipeline from scrapy.spider import spiders from scrapy.spider.middleware import SpiderMiddlewareManager -from scrapy.utils.misc import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate +from scrapy.utils.defer import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate from scrapy.conf import settings diff --git a/scrapy/trunk/scrapy/core/manager.py b/scrapy/trunk/scrapy/core/manager.py index 2719172c7..074dcdb2a 100644 --- a/scrapy/trunk/scrapy/core/manager.py +++ b/scrapy/trunk/scrapy/core/manager.py @@ -5,7 +5,7 @@ from scrapy.core import log from scrapy.http import Request from scrapy.core.engine import scrapyengine from scrapy.spider import spiders -from scrapy.utils.misc import load_class +from scrapy.utils.defer import load_class from scrapy.utils.url import is_url from scrapy.conf import settings diff --git a/scrapy/trunk/scrapy/core/scheduler/schedulers.py b/scrapy/trunk/scrapy/core/scheduler/schedulers.py index 40cf7b5ef..c01beb329 100644 --- a/scrapy/trunk/scrapy/core/scheduler/schedulers.py +++ b/scrapy/trunk/scrapy/core/scheduler/schedulers.py @@ -8,7 +8,7 @@ from scrapy.core.scheduler.filter import GroupFilter from scrapy.core import log from scrapy.core.exceptions import IgnoreRequest from scrapy.utils.datatypes import PriorityQueue, PriorityStack -from scrapy.utils.misc import defer_fail +from scrapy.utils.defer import defer_fail from scrapy.conf import settings diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index 4723d01c0..5e1765f28 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -10,7 +10,7 @@ from scrapy.http.url import Url from scrapy.http.headers import Headers from scrapy.utils.url import safe_url_string from scrapy.utils.c14n import canonicalize -from scrapy.utils.misc import chain_deferred +from scrapy.utils.defer import chain_deferred class Request(object): def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None, diff --git a/scrapy/trunk/scrapy/item/pipeline.py b/scrapy/trunk/scrapy/item/pipeline.py index 202d5c596..a17e2b69d 100644 --- a/scrapy/trunk/scrapy/item/pipeline.py +++ b/scrapy/trunk/scrapy/item/pipeline.py @@ -1,7 +1,8 @@ from scrapy.core import log from scrapy.core.exceptions import DropItem, NotConfigured from scrapy.item import ScrapedItem -from scrapy.utils.misc import load_class, defer_succeed, mustbe_deferred +from scrapy.utils.misc import load_class +from scrapy.utils.defer import defer_succeed, mustbe_deferred from scrapy.conf import settings class ItemPipeline(object): diff --git a/scrapy/trunk/scrapy/spider/middleware.py b/scrapy/trunk/scrapy/spider/middleware.py index b3a5cbe8f..70d25d1c2 100644 --- a/scrapy/trunk/scrapy/spider/middleware.py +++ b/scrapy/trunk/scrapy/spider/middleware.py @@ -7,7 +7,8 @@ from twisted.python.failure import Failure from scrapy.core import log from scrapy.core.exceptions import NotConfigured -from scrapy.utils.misc import load_class, mustbe_deferred +from scrapy.utils.misc import load_class +from scrapy.utils.defer import mustbe_deferred from scrapy.conf import settings def _isiterable(possible_iterator): diff --git a/scrapy/trunk/scrapy/utils/defer.py b/scrapy/trunk/scrapy/utils/defer.py new file mode 100644 index 000000000..453d7320d --- /dev/null +++ b/scrapy/trunk/scrapy/utils/defer.py @@ -0,0 +1,81 @@ +""" +Helper functions for dealing with Twisted deferreds +""" + +from twisted.internet import defer, reactor +from twisted.python import failure + +def defer_fail(_failure): + """same as twsited.internet.defer.fail, but delay calling errback """ + d = defer.Deferred() + reactor.callLater(0, d.errback, _failure) + return d + +def defer_succeed(result): + """same as twsited.internet.defer.succed, but delay calling callback""" + d = defer.Deferred() + reactor.callLater(0, d.callback, result) + return d + +def defer_result(result): + if isinstance(result, defer.Deferred): + return result + elif isinstance(result, failure.Failure): + return defer_fail(result) + else: + return defer_succeed(result) + +def mustbe_deferred(f, *args, **kw): + """same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback""" + try: + result = f(*args, **kw) + except: + return defer_fail(failure.Failure()) + else: + return defer_result(result) + +def chain_deferred(d1, d2): + if callable(d2): + d2 = lambda_deferred(d2) + + def _pause(_): + d2.pause() + reactor.callLater(0, d2.unpause) + return _ + + def _reclaim(_): + return d2 + + #d1.addBoth(_pause) ## needs more debugging before reenable it + d1.chainDeferred(d2) + d1.addBoth(_reclaim) + return d1 + +def lambda_deferred(func): + deferred = defer.Deferred() + def _success(res): + d = func() + d.callback(res) + return d + def _fail(res): + d = func() + d.errback(res) + return d + return deferred.addCallbacks(_success, _fail) + +def deferred_degenerate(generator, container=None, next_delay=0): + generator = iter(generator or []) + deferred = defer.Deferred() + container = container or [] + def _next(): + try: + container.append(generator.next()) + except StopIteration: + reactor.callLater(0, deferred.callback, container) + except: + reactor.callLater(0, deferred.errback, failure.Failure()) + else: + reactor.callLater(next_delay, _next) + _next() + return deferred + diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index b567d3f67..5a3f34105 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -3,12 +3,12 @@ Auxiliary functions which doesn't fit anywhere else """ import re -from twisted.internet import defer, reactor -from twisted.python import failure +from twisted.internet import defer from scrapy.core.exceptions import UsageError from scrapy.utils.python import flatten from scrapy.utils.markup import remove_entities +from scrapy.utils.defer import defer_succeed def dict_updatedefault(D, E, **F): """ @@ -28,69 +28,6 @@ def dict_updatedefault(D, E, **F): for k in F: D.setdefault(k, F[k]) - -def defer_fail(_failure): - """same as twsited.internet.defer.fail, but delay calling errback """ - d = defer.Deferred() - reactor.callLater(0, d.errback, _failure) - return d - - -def defer_succeed(result): - """same as twsited.internet.defer.succed, but delay calling callback""" - d = defer.Deferred() - reactor.callLater(0, d.callback, result) - return d - -def defer_result(result): - if isinstance(result, defer.Deferred): - return result - elif isinstance(result, failure.Failure): - return defer_fail(result) - else: - return defer_succeed(result) - -def mustbe_deferred(f, *args, **kw): - """same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback""" - try: - result = f(*args, **kw) - except: - return defer_fail(failure.Failure()) - else: - return defer_result(result) - - -def chain_deferred(d1, d2): - if callable(d2): - d2 = lambda_deferred(d2) - - def _pause(_): - d2.pause() - reactor.callLater(0, d2.unpause) - return _ - - def _reclaim(_): - return d2 - - #d1.addBoth(_pause) ## needs more debugging before reenable it - d1.chainDeferred(d2) - d1.addBoth(_reclaim) - return d1 - - -def lambda_deferred(func): - deferred = defer.Deferred() - def _success(res): - d = func() - d.callback(res) - return d - def _fail(res): - d = func() - d.errback(res) - return d - return deferred.addCallbacks(_success, _fail) - - def memoize(cache, hash): def decorator(func): def wrapper(*args, **kwargs): @@ -110,24 +47,6 @@ def memoize(cache, hash): return wrapper return decorator - -def deferred_degenerate(generator, container=None, next_delay=0): - generator = iter(generator or []) - deferred = defer.Deferred() - container = container or [] - def _next(): - try: - container.append(generator.next()) - except StopIteration: - reactor.callLater(0, deferred.callback, container) - except: - reactor.callLater(0, deferred.errback, failure.Failure()) - else: - reactor.callLater(next_delay, _next) - _next() - return deferred - - def stats_getpath(dict_, path, default=None): for key in path.split('/'): if key in dict_: