Add utils.defer.isasyncgen().

docs/topics/coroutines.rst

@@ -100,6 +100,8 @@ synchronous :meth:`~scrapy.spiders.Spider.start_requests`.
 Another option is to make separate methods for normal and asynchronous
 iterables and choose one at run time::
 
+    from scrapy.utils.defer import isasyncgen
+
     class ProcessStartRequestsAsyncGenMiddleware:
         def _normal_process_start_requests(self, start_requests, spider):
             # ... do something with normal start_requests
@@ -108,7 +110,7 @@ iterables and choose one at run time::
             # ... do something with async start_requests
 
         def process_start_requests(self, start_requests, spider):
-            if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(start_requests):
+            if isasyncgen(start_requests):
                 return self._async_process_start_requests(start_requests, spider)
             else:
                 return self._normal_process_start_requests(start_requests, spider)

scrapy/core/engine.py

@@ -4,7 +4,6 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
 For more information see docs/topics/architecture.rst
 
 """
-import inspect
 import logging
 from time import time
 
@@ -15,7 +14,7 @@ from scrapy import signals
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Response, Request
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, isasyncgen
 from scrapy.utils.misc import load_object
 from scrapy.utils.reactor import CallLaterOnce
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
@@ -28,7 +27,7 @@ class Slot:
     def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
         self.closing = False
         self.inprogress = set()  # requests in progress
-        if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(start_requests):
+        if isasyncgen(start_requests):
             self.start_requests = start_requests
         else:
             self.start_requests = iter(start_requests)
@@ -132,7 +131,7 @@ class ExecutionEngine:
 
         if slot.start_requests and not self._needs_backout(spider):
             try:
-                if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(slot.start_requests):
+                if isasyncgen(slot.start_requests):
                     request = yield deferred_from_coro(slot.start_requests.__anext__())
                 else:
                     request = next(slot.start_requests)

scrapy/utils/defer.py

@@ -170,3 +170,10 @@ def maybeDeferred_coro(f, *args, **kw):
         return defer.fail(result)
     else:
         return defer.succeed(result)
+
+
+def isasyncgen(o):
+    """ Returns inspect.isasyncgen() result if it's available (requires Python 3.6),
+    otherwise returns False.
+    """
+    return hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(o)

scrapy/utils/middlewares.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-import inspect
+from scrapy.utils.defer import isasyncgen
 
 
 def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
@@ -18,7 +18,7 @@ def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor
     For each item in the iterable: skips it if in_predicate is False, applies processor,
     skips the result if out_predicate is False, else yields it.
     """
-    if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(it):
+    if isasyncgen(it):
         from scrapy.utils.asyncgen import process_async_iterable_helper  # Python 3.5 limitation
         return process_async_iterable_helper(it, in_predicate, out_predicate, processor)
     else:

scrapy/utils/spider.py

@@ -2,7 +2,7 @@ import logging
 import inspect
 
 from scrapy.spiders import Spider
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, isasyncgen
 from scrapy.utils.misc import arg_to_iter
 try:
     from scrapy.utils.asyncgen import collect_asyncgen
@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
-    if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
+    if collect_asyncgen and isasyncgen(result):
         d = deferred_from_coro(collect_asyncgen(result))
         d.addCallback(iterate_spider_output)
         return d