diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3cf005e28..52ac9e209 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -100,6 +100,8 @@ synchronous :meth:`~scrapy.spiders.Spider.start_requests`. Another option is to make separate methods for normal and asynchronous iterables and choose one at run time:: + from scrapy.utils.defer import isasyncgen + class ProcessStartRequestsAsyncGenMiddleware: def _normal_process_start_requests(self, start_requests, spider): # ... do something with normal start_requests @@ -108,7 +110,7 @@ iterables and choose one at run time:: # ... do something with async start_requests def process_start_requests(self, start_requests, spider): - if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(start_requests): + if isasyncgen(start_requests): return self._async_process_start_requests(start_requests, spider) else: return self._normal_process_start_requests(start_requests, spider) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 658758d94..2f0179ac9 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,7 +4,6 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. For more information see docs/topics/architecture.rst """ -import inspect import logging from time import time @@ -15,7 +14,7 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider from scrapy.http import Response, Request -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_from_coro, isasyncgen from scrapy.utils.misc import load_object from scrapy.utils.reactor import CallLaterOnce from scrapy.utils.log import logformatter_adapter, failure_to_exc_info @@ -28,7 +27,7 @@ class Slot: def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False self.inprogress = set() # requests in progress - if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(start_requests): + if isasyncgen(start_requests): self.start_requests = start_requests else: self.start_requests = iter(start_requests) @@ -132,7 +131,7 @@ class ExecutionEngine: if slot.start_requests and not self._needs_backout(spider): try: - if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(slot.start_requests): + if isasyncgen(slot.start_requests): request = yield deferred_from_coro(slot.start_requests.__anext__()) else: request = next(slot.start_requests) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 34b8d9774..e204b8aa8 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -170,3 +170,10 @@ def maybeDeferred_coro(f, *args, **kw): return defer.fail(result) else: return defer.succeed(result) + + +def isasyncgen(o): + """ Returns inspect.isasyncgen() result if it's available (requires Python 3.6), + otherwise returns False. + """ + return hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(o) diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py index e60cea8f9..fdde3c9a2 100644 --- a/scrapy/utils/middlewares.py +++ b/scrapy/utils/middlewares.py @@ -1,5 +1,5 @@ # coding: utf-8 -import inspect +from scrapy.utils.defer import isasyncgen def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): @@ -18,7 +18,7 @@ def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor For each item in the iterable: skips it if in_predicate is False, applies processor, skips the result if out_predicate is False, else yields it. """ - if hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(it): + if isasyncgen(it): from scrapy.utils.asyncgen import process_async_iterable_helper # Python 3.5 limitation return process_async_iterable_helper(it, in_predicate, out_predicate, processor) else: diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 59038ef66..73bec671f 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,7 +2,7 @@ import logging import inspect from scrapy.spiders import Spider -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_from_coro, isasyncgen from scrapy.utils.misc import arg_to_iter try: from scrapy.utils.asyncgen import collect_asyncgen @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + if collect_asyncgen and isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d