diff --git a/conftest.py b/conftest.py index 5737dae00..ba169cf87 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,4 @@ +import sys from pathlib import Path import pytest @@ -8,8 +9,6 @@ def _py_files(folder): collect_ignore = [ - # no tests and not importable on 3.5 - "scrapy/utils/asyncgen.py", # not a test, but looks like a test "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess @@ -20,6 +19,10 @@ collect_ignore = [ *_py_files("tests/py36"), ] +if sys.version_info < (3, 6): + # not importable on 3.5 + collect_ignore.append("scrapy/utils/asyncgen.py") + for line in open('tests/ignores.txt'): file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index d4f9af86b..3cf005e28 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -44,7 +44,8 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` - method of :ref:`spider middlewares `. + method of :ref:`spider middlewares `. See + :ref:`async-start_requests` below for details. - :ref:`Signal handlers that support deferreds `. @@ -57,11 +58,17 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): Asynchronous start_requests and spider middlewares ================================================== -.. versionadded:: 2.1 +.. versionadded:: 2.2 The :meth:`~scrapy.spiders.Spider.start_requests` spider method can be an -asynchronous generator. In this case all spider middlewares used with this -spider that have the +asynchronous generator:: + + async def start_requests(): + # ... + yield scrapy.Request(...) + # ... + +In this case all spider middlewares used with this spider that have the :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method must support this: if they receive an asynchronous iterable, they must return one as well. On the other hand, if they receive a normal iterable, they @@ -83,15 +90,15 @@ Here is an example of a universal middleware using this approach:: class ProcessStartRequestsAsyncGenMiddleware: async def process_start_requests(self, start_requests, spider): - async for r in as_async_generator(start_requests): - # ... do something with r - yield r + async for req in as_async_generator(start_requests): + # ... do something with req + yield req If this method includes asynchronous code, that code will work even with synchronous :meth:`~scrapy.spiders.Spider.start_requests`. Another option is to make separate methods for normal and asynchronous -iterables and choose one at the run time:: +iterables and choose one at run time:: class ProcessStartRequestsAsyncGenMiddleware: def _normal_process_start_requests(self, start_requests, spider): diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0cf3e1ea8..0e214a40a 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -148,8 +148,8 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_start_requests(start_requests, spider) .. versionadded:: 0.15 - .. versionchanged:: 2.1 - Since 2.1 this can take and return an :term:`python:asynchronous + .. versionchanged:: 2.2 + Since 2.2 this can take and return an :term:`python:asynchronous iterable`. This method is called with the start requests of the spider, and works @@ -165,7 +165,7 @@ object gives you access, for example, to the :ref:`settings `. middleware method should be an :term:`python:asynchronous generator`, which should support both synchronous and asynchronous ``start_requests``. :func:`scrapy.utils.asyncgen.as_async_generator` - can be used to convert both kinds of ``start_requests`` to an + can be used to convert both kinds of ``start_requests`` into an asynchronous iterable. .. note:: When implementing this method in your spider middleware, you