added scrapy.utils.defer, moved deferred functions from scrapy.utils.misc to that module

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40271

scrapy/trunk/scrapy/contrib/pipeline/media.py

@@ -1,6 +1,6 @@
 from twisted.internet import defer
 
-from scrapy.utils.misc import mustbe_deferred, defer_result
+from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.core import log
 from scrapy.core.engine import scrapyengine
 from scrapy.core.exceptions import DropItem, NotConfigured

scrapy/trunk/scrapy/contrib/web/json.py

@@ -1,6 +1,6 @@
 from functools import wraps
 from twisted.internet import defer
-from scrapy.utils.misc import mustbe_deferred
+from scrapy.utils.defer import mustbe_deferred
 
 from .http import HttpResponse
 

scrapy/trunk/scrapy/contrib/web/site.py

@@ -61,7 +61,7 @@ import re
 
 from twisted.web import server, resource
 from twisted.internet import reactor
-from scrapy.utils.misc import mustbe_deferred
+from scrapy.utils.defer import mustbe_deferred
 
 from .http import HttpResponse, build_httprequest
 

scrapy/trunk/scrapy/core/downloader/handlers.py

@@ -13,7 +13,7 @@ from twisted.web import error as web_error
 from scrapy.core import signals
 from scrapy.http import Request, Response, Headers
 from scrapy.core.exceptions import UsageError, HttpException
-from scrapy.utils.misc import defer_succeed
+from scrapy.utils.defer import defer_succeed
 from scrapy.conf import settings
 
 def download_any(request, spider):

scrapy/trunk/scrapy/core/downloader/manager.py

@@ -12,7 +12,7 @@ from scrapy.core.downloader.handlers import download_any
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.core import log
 from scrapy.conf import settings
-from scrapy.utils.misc import chain_deferred
+from scrapy.utils.defer import chain_deferred, mustbe_deferred
 
 
 class SiteDetails(object):
@@ -126,7 +126,6 @@ class Downloader(object):
         def _finish(result):
             self.process_queue(spider)
 
-        from scrapy.utils.misc import mustbe_deferred
         dwld = mustbe_deferred(self.download_function, request, spider)
         dwld.addBoth(_remove)
         chain_deferred(dwld, deferred)

scrapy/trunk/scrapy/core/downloader/middleware.py

@@ -4,7 +4,8 @@ request-response middleware extension
 from scrapy.core import signals, log
 from scrapy.http import Request, Response
 from scrapy.core.exceptions import NotConfigured
-from scrapy.utils.misc import load_class, mustbe_deferred
+from scrapy.utils.misc import load_class
+from scrapy.utils.defer import mustbe_deferred
 from scrapy.core.downloader.handlers import download_any
 from scrapy.conf import settings
 

scrapy/trunk/scrapy/core/engine.py

@@ -17,7 +17,7 @@ from scrapy.item import ScrapedItem
 from scrapy.item.pipeline import ItemPipeline
 from scrapy.spider import spiders
 from scrapy.spider.middleware import SpiderMiddlewareManager
-from scrapy.utils.misc import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate
+from scrapy.utils.defer import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate
 from scrapy.conf import settings
 
 

scrapy/trunk/scrapy/core/manager.py

@@ -5,7 +5,7 @@ from scrapy.core import log
 from scrapy.http import Request
 from scrapy.core.engine import scrapyengine
 from scrapy.spider import spiders
-from scrapy.utils.misc import load_class
+from scrapy.utils.defer import load_class
 from scrapy.utils.url import is_url
 from scrapy.conf import settings
 

scrapy/trunk/scrapy/core/scheduler/schedulers.py

@@ -8,7 +8,7 @@ from scrapy.core.scheduler.filter import GroupFilter
 from scrapy.core import log
 from scrapy.core.exceptions import IgnoreRequest
 from scrapy.utils.datatypes import PriorityQueue, PriorityStack
-from scrapy.utils.misc import defer_fail
+from scrapy.utils.defer import defer_fail
 from scrapy.conf import settings
 
 

scrapy/trunk/scrapy/http/request.py

@@ -10,7 +10,7 @@ from scrapy.http.url import Url
 from scrapy.http.headers import Headers
 from scrapy.utils.url import safe_url_string
 from scrapy.utils.c14n import canonicalize
-from scrapy.utils.misc import chain_deferred
+from scrapy.utils.defer import chain_deferred
 
 class Request(object):
     def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None,

scrapy/trunk/scrapy/item/pipeline.py

@@ -1,7 +1,8 @@
 from scrapy.core import log
 from scrapy.core.exceptions import DropItem, NotConfigured
 from scrapy.item import ScrapedItem
-from scrapy.utils.misc import load_class, defer_succeed, mustbe_deferred
+from scrapy.utils.misc import load_class
+from scrapy.utils.defer import defer_succeed, mustbe_deferred
 from scrapy.conf import settings
 
 class ItemPipeline(object):

scrapy/trunk/scrapy/spider/middleware.py

@@ -7,7 +7,8 @@ from twisted.python.failure import Failure
 
 from scrapy.core import log
 from scrapy.core.exceptions import NotConfigured
-from scrapy.utils.misc import load_class, mustbe_deferred
+from scrapy.utils.misc import load_class
+from scrapy.utils.defer import mustbe_deferred
 from scrapy.conf import settings
 
 def _isiterable(possible_iterator):

scrapy/trunk/scrapy/utils/defer.py

@@ -0,0 +1,81 @@
+"""
+Helper functions for dealing with Twisted deferreds
+"""
+
+from twisted.internet import defer, reactor
+from twisted.python import failure
+
+def defer_fail(_failure):
+    """same as twsited.internet.defer.fail, but delay calling errback """
+    d = defer.Deferred()
+    reactor.callLater(0, d.errback, _failure)
+    return d
+
+def defer_succeed(result):
+    """same as twsited.internet.defer.succed, but delay calling callback"""
+    d = defer.Deferred()
+    reactor.callLater(0, d.callback, result)
+    return d
+
+def defer_result(result):
+    if isinstance(result, defer.Deferred):
+        return result
+    elif isinstance(result, failure.Failure):
+        return defer_fail(result)
+    else:
+        return defer_succeed(result)
+
+def mustbe_deferred(f, *args, **kw):
+    """same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback"""
+    try:
+        result = f(*args, **kw)
+    except:
+        return defer_fail(failure.Failure())
+    else:
+        return defer_result(result)
+
+def chain_deferred(d1, d2):
+    if callable(d2):
+        d2 = lambda_deferred(d2)
+
+    def _pause(_):
+        d2.pause()
+        reactor.callLater(0, d2.unpause)
+        return _
+
+    def _reclaim(_):
+        return d2
+
+    #d1.addBoth(_pause) ## needs more debugging before reenable it
+    d1.chainDeferred(d2)
+    d1.addBoth(_reclaim)
+    return d1
+
+def lambda_deferred(func):
+    deferred = defer.Deferred()
+    def _success(res):
+        d = func()
+        d.callback(res)
+        return d
+    def _fail(res):
+        d = func()
+        d.errback(res)
+        return d
+    return deferred.addCallbacks(_success, _fail)
+
+def deferred_degenerate(generator, container=None, next_delay=0):
+    generator = iter(generator or [])
+    deferred = defer.Deferred()
+    container = container or []
+    def _next():
+        try:
+            container.append(generator.next())
+        except StopIteration:
+            reactor.callLater(0, deferred.callback, container)
+        except:
+            reactor.callLater(0, deferred.errback, failure.Failure())
+        else:
+            reactor.callLater(next_delay, _next)
+    _next()
+    return deferred
+

scrapy/trunk/scrapy/utils/misc.py

@@ -3,12 +3,12 @@ Auxiliary functions which doesn't fit anywhere else
 """
 import re
 
-from twisted.internet import defer, reactor
+from twisted.internet import defer
-from twisted.python import failure
 
 from scrapy.core.exceptions import UsageError
 from scrapy.utils.python import flatten
 from scrapy.utils.markup import remove_entities
+from scrapy.utils.defer import defer_succeed
 
 def dict_updatedefault(D, E, **F):
     """
@@ -28,69 +28,6 @@ def dict_updatedefault(D, E, **F):
     for k in F:
         D.setdefault(k, F[k])
 
-
-def defer_fail(_failure):
-    """same as twsited.internet.defer.fail, but delay calling errback """
-    d = defer.Deferred()
-    reactor.callLater(0, d.errback, _failure)
-    return d
-
-
-def defer_succeed(result):
-    """same as twsited.internet.defer.succed, but delay calling callback"""
-    d = defer.Deferred()
-    reactor.callLater(0, d.callback, result)
-    return d
-
-def defer_result(result):
-    if isinstance(result, defer.Deferred):
-        return result
-    elif isinstance(result, failure.Failure):
-        return defer_fail(result)
-    else:
-        return defer_succeed(result)
-
-def mustbe_deferred(f, *args, **kw):
-    """same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback"""
-    try:
-        result = f(*args, **kw)
-    except:
-        return defer_fail(failure.Failure())
-    else:
-        return defer_result(result)
-
-
-def chain_deferred(d1, d2):
-    if callable(d2):
-        d2 = lambda_deferred(d2)
-
-    def _pause(_):
-        d2.pause()
-        reactor.callLater(0, d2.unpause)
-        return _
-
-    def _reclaim(_):
-        return d2
-
-    #d1.addBoth(_pause) ## needs more debugging before reenable it
-    d1.chainDeferred(d2)
-    d1.addBoth(_reclaim)
-    return d1
-
-
-def lambda_deferred(func):
-    deferred = defer.Deferred()
-    def _success(res):
-        d = func()
-        d.callback(res)
-        return d
-    def _fail(res):
-        d = func()
-        d.errback(res)
-        return d
-    return deferred.addCallbacks(_success, _fail)
-
-
 def memoize(cache, hash):
     def decorator(func):
         def wrapper(*args, **kwargs):
@@ -110,24 +47,6 @@ def memoize(cache, hash):
         return wrapper
     return decorator
 
-
-def deferred_degenerate(generator, container=None, next_delay=0):
-    generator = iter(generator or [])
-    deferred = defer.Deferred()
-    container = container or []
-    def _next():
-        try:
-            container.append(generator.next())
-        except StopIteration:
-            reactor.callLater(0, deferred.callback, container)
-        except:
-            reactor.callLater(0, deferred.errback, failure.Failure())
-        else:
-            reactor.callLater(next_delay, _next)
-    _next()
-    return deferred
-
-
 def stats_getpath(dict_, path, default=None):
     for key in path.split('/'):
         if key in dict_: