Add async def support to signal handlers that already supported Deferreds.

scrapy/utils/defer.py

@@ -140,3 +140,20 @@ def deferred_from_coro(o):
             # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
             return defer.Deferred.fromFuture(asyncio.ensure_future(o))
     return o
+
+
+def maybeDeferred_coro(f, *args, **kw):
+    """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
+    try:
+        result = f(*args, **kw)
+    except:  # noqa: E722
+        return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))
+
+    if isinstance(result, defer.Deferred):
+        return result
+    elif _isfuture(result) or inspect.isawaitable(result):
+        return deferred_from_coro(result)
+    elif isinstance(result, failure.Failure):
+        return defer.fail(result)
+    else:
+        return defer.succeed(result)

scrapy/utils/signal.py

@@ -2,12 +2,14 @@
 
 import logging
 
-from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
+from twisted.internet.defer import DeferredList, Deferred
 from twisted.python.failure import Failure
 
 from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
     getAllReceivers, disconnect
 from pydispatch.robustapply import robustApply
+
+from scrapy.utils.defer import maybeDeferred_coro
 from scrapy.utils.log import failure_to_exc_info
 
 logger = logging.getLogger(__name__)
@@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     spider = named.get('spider', None)
     dfds = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender,
+        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
                 *arguments, **named)
         d.addErrback(logerror, receiver)
         d.addBoth(lambda result: (receiver, result))

tests/test_utils_signal.py

@@ -1,3 +1,6 @@
+import asyncio
+
+from pytest import mark
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -5,6 +8,7 @@ from twisted.internet import defer, reactor
 from pydispatch import dispatcher
 
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
+from scrapy.utils.test import get_from_asyncio_queue
 
 
 class SendCatchLogTest(unittest.TestCase):
@@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest):
         return send_catch_log_deferred(signal, *a, **kw)
 
 
-class SendCatchLogDeferredTest2(SendCatchLogTest):
+class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
@@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
         reactor.callLater(0, d.callback, "OK")
         return d
 
-    def _get_result(self, signal, *a, **kw):
-        return send_catch_log_deferred(signal, *a, **kw)
+
+class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
+
+    async def ok_handler(self, arg, handlers_called):
+        handlers_called.add(self.ok_handler)
+        assert arg == 'test'
+        await defer.succeed(42)
+        return "OK"
+
+
+@mark.only_asyncio()
+class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
+
+    async def ok_handler(self, arg, handlers_called):
+        handlers_called.add(self.ok_handler)
+        assert arg == 'test'
+        await asyncio.sleep(0.2)
+        return await get_from_asyncio_queue("OK")
 
 
 class SendCatchLogTest2(unittest.TestCase):