diff --git a/docs/contributing.rst b/docs/contributing.rst index f5c1c74b8..bb197b428 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -228,10 +228,9 @@ with a name of the branch you want to create locally). See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. -E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" -prefer "Fix hanging when exception occurs in start_requests (#411)" -instead of "Fix for #411". Complete titles make it easy to skim through -the issue tracker. +E.g. For bug #411: "Scrapy hangs if an exception raises in yield_seeds" prefer +"Fix hanging when exception occurs in yield_seeds (#411)" instead of "Fix for +#411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits from functional changes. This will make pull diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6e6caebf1..8f74f76af 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def yield_seeds(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.yield_seeds`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -164,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as an argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.yield_seeds` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``yield_seeds`` method +---------------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.yield_seeds` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.yield_seeds` to create the initial requests for your +spider. .. code-block:: python @@ -292,7 +293,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -302,8 +303,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon @@ -794,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def yield_seeds(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 8230c3aef..28ee6f023 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,82 @@ Release notes ============= +.. _release-VERSION: + +Scrapy VERSION (unreleased) +--------------------------- + +Highlights: + +- Replaced ``start_requests`` with :meth:`~scrapy.Spider.yield_seeds` + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``scrapy.core.engine.Slot.start_requests` and its matching + ``Slot.__init__()`` parameter have been removed, replaced by + ``seeds_iterator``. + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.Spider.start_requests`` is deprecated, use + :meth:`~scrapy.Spider.yield_seeds` instead. + + To make spiders compatible with older Scrapy versions while avoiding a + deprecation warning on Scrapy VERSION and higher, you can define both + methods. For example: + + .. code-block:: python + + async def yield_seeds(self) -> AsyncIterator[Any]: + for seed in self.start_requests(): + yield seed + + + def start_requests(self) -> Iterable[Request]: + yield Request(url="https://toscrape.com") + + (:issue:`456`, :issue:`3237`, :issue:`5627`, …) + +- The ``process_start_requests`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seed` instead. + + Defining both methods is OK, e.g. to support older Scrapy versions, but + only :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` is + used by Scrapy VERSION and higher. + + (:issue:`456`, :issue:`3237`, :issue:`5627`, …) + +- The ``scrapy.spiders.init.InitSpider`` spider class is deprecated. + +.. + TODO: Update the related issues lists including #456 to include other + related issues. + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.yield_seeds` asynchronous generator. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without the need to use a workaround such as + yielding the start requests and items from a spider callback instead. + + (:issue:`456`, :issue:`3237`, :issue:`5627`, …) + +- The new :setting:`SEEDING_POLICY` setting allows customizing how spider + start requests and items are consumed. + + Additionally, related new settings have been added: + :setting:`SEEDING_INITIAL_TIMEOUT`, :setting:`SEEDING_TIMEOUT`. + + (:issue:`456`, :issue:`3237`, :issue:`5627`, …) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) @@ -12,7 +88,7 @@ Highlights: - Dropped support for Python 3.8, added support for Python 3.13 -- :meth:`~scrapy.Spider.start_requests` can now yield items +- ``scrapy.Spider.start_requests`` can now yield items - Added :class:`~scrapy.http.JsonResponse` @@ -303,7 +379,7 @@ Deprecations New features ~~~~~~~~~~~~ -- :meth:`~scrapy.Spider.start_requests` can now yield items. +- ``scrapy.Spider.start_requests`` can now yield items. (:issue:`5289`, :issue:`6417`) - Added a new :class:`~scrapy.http.Response` subclass, @@ -795,7 +871,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests` or in a handler of the + ``scrapy.Spider.start_requests`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now @@ -934,10 +1010,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals @@ -3371,7 +3447,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -6292,7 +6368,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -6485,7 +6561,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -6548,7 +6624,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) diff --git a/docs/requirements.txt b/docs/requirements.txt index 103fb08d6..038530c76 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ sphinx==8.1.3 -sphinx-hoverxref==1.4.2 +# https://github.com/readthedocs/sphinx-hoverxref/issues/312 +sphinx-hoverxref @ git+https://github.com/readthedocs/sphinx-hoverxref.git@f702d3efffda796ba1d7747b64cadb732fa8a274 sphinx-notfound-page==1.0.4 sphinx-rtd-theme==3.0.2 sphinx-rtd-dark-mode==1.3.0 diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0370dc538..4eff3cbe7 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process seed requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 57aa3a62d..a7ba5443f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -6,8 +6,8 @@ Coroutines .. versionadded:: 2.0 -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. +Scrapy has :ref:`partial support ` for the :ref:`coroutine +syntax ` (i.e. ``async def``). .. _coroutine-support: @@ -17,6 +17,10 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): +- The :meth:`~scrapy.spiders.Spider.yield_seeds` spider method. + + .. versionadded: VERSION + - :class:`~scrapy.Request` callbacks. If you are using any custom or third-party :ref:`spider middleware @@ -37,8 +41,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method of :ref:`spider middlewares `. @@ -51,6 +53,13 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` method + of :ref:`spider middlewares `. + + .. versionadded:: VERSION + +- :ref:`Signal handlers that support deferreds `. + General usage ============= @@ -123,8 +132,9 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.spiders.Spider.yield_seeds`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b187f3aaf..f06643c37 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,23 +31,12 @@ Request objects If the URL is invalid, a :exc:`ValueError` exception is raised. :type url: str - :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. + :param callback: sets :attr:`callback`, defaults to ``None``. - In addition to a function, the following values are supported: - - - ``None`` (default), which indicates that the spider's - :meth:`~scrapy.Spider.parse` method must be used. - - - :func:`~scrapy.http.request.NO_CALLBACK` - - For more information, see - :ref:`topics-request-response-ref-request-callback-arguments`. - - .. note:: If exceptions are raised during processing, ``errback`` is - called instead. - - :type callback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type callback: Callable[Concatenate[Response, ...], Any] | None :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str @@ -144,23 +133,15 @@ Request objects Negative values are allowed in order to indicate relatively low-priority. :type priority: int - :param dont_filter: indicates that this request should not be filtered by - the scheduler or some middlewares. This is used when you want to perform - an identical request multiple times, to ignore the duplicates filter. - Use it with care, or you will get into crawling loops. Default to ``False``. + :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. :type dont_filter: bool - :param errback: a function that will be called if any exception was - raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a - :exc:`~twisted.python.failure.Failure` as first parameter. - For more information, - see :ref:`topics-request-response-ref-errbacks` below. + :param errback: sets :attr:`errback`, defaults to ``None``. - .. versionchanged:: 2.0 - The *callback* parameter is no longer required when the *errback* - parameter is specified. - :type errback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type errback: Callable[[Failure], Any] | None :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -194,6 +175,25 @@ Request objects This attribute is read-only. To change the body of a Request use :meth:`replace`. + .. autoattribute:: callback + + .. autoattribute:: errback + + .. attribute:: Request.cb_kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. + + This dict is :doc:`shallow copied ` when the request is + cloned using the ``copy()`` or ``replace()`` methods, and can also be + accessed, in your spider, from the ``response.cb_kwargs`` attribute. + + In case of a failure to process the request, this dict can be accessed as + ``failure.request.cb_kwargs`` in the request's errback. For more information, + see :ref:`errback-cb_kwargs`. + .. attribute:: Request.meta :value: {} @@ -237,20 +237,7 @@ Request objects Also mind that the :meth:`copy` and :meth:`replace` request methods :doc:`shallow-copy ` request metadata. - .. attribute:: Request.cb_kwargs - - A dictionary that contains arbitrary metadata for this request. Its contents - will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a - :class:`~scrapy.http.Response` object as argument. - - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.cb_kwargs`` attribute. - - In case of a failure to process the request, this dict can be accessed as - ``failure.request.cb_kwargs`` in the request's errback. For more information, - see :ref:`errback-cb_kwargs`. + .. autoattribute:: dont_filter .. autoattribute:: Request.attributes @@ -366,7 +353,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - def start_requests(self): + async def yield_seeds(self): for u in self.start_urls: yield scrapy.Request( u, @@ -1309,7 +1296,7 @@ JsonResponse objects .. class:: JsonResponse(url[, ...]) - The :class:`JsonResponse` class is a subclass of :class:`TextResponse` - that is used when the response has a `JSON MIME type - `_ in its `Content-Type` + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` header. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7646aca4f..d04950a4a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1752,6 +1752,74 @@ Soft limit (in bytes) for response data being processed. While the sum of the sizes of all responses being processed is above this value, Scrapy does not process new requests. +.. setting:: SEEDING_POLICY + +SEEDING_POLICY +-------------- + +.. versionadded:: VERSION + +Default: ``"lazy"`` + +The way :meth:`Spider.yield_seeds ` is iterated: + +- .. _lazy-seeding: + + ``"lazy"``: Seeds are only read while the :ref:`scheduler + ` is empty and the number of ongoing requests is + lower than :setting:`CONCURRENT_REQUESTS`. + + This seeding policy aims to: + + - Maximize crawl speed by maxing out concurrent requests as often + as possible. + + - Minimize the number of requests in the scheduler at any given + time by prioritizing scheduler requests over seeds, to minimize + resource usage (memory or disk, depending on + :setting:`JOBDIR`). + + This seeding policy is best used when seed request priority is not + important. Switching to :ref:`serial ` may lower + resource usage further at the cost of also lowering crawl speed. + +- .. _front-load-seeding: + + ``"front-load"``: The spider does not start until all seeds have + been read and loaded into the scheduler. + + This seeding policy aims to give the :ref:`scheduler + ` full control over request order, at the cost of + a higher resource usage and a delayed crawl start. + + This seeding policy is best used when having all requests go + through the scheduler is more important than resource usage and + crawl speed. + +- .. _greedy-seeding: + + ``"greedy"``: While the :ref:`scheduler ` is + empty and the number of ongoing requests is lower than + :setting:`CONCURRENT_REQUESTS`, seeds are read and sent directly + (bypassing the scheduler). While the scheduler has requests, seeds + are fed into the scheduler. + + This seeding policy is similar to :ref:`front-load + `, but it bypasses the scheduler for the first + few requests to avoid delaying the crawl start. + +- .. _serial-seeding: + + ``"serial"``: A single seed is read whenever the :ref:`scheduler + ` is empty and there are no ongoing requests. + That is, a new seed is not read until all requests triggered by the + previous seed, directly or indirectly, have been processed. + + This seeding policy is similar to :ref:`lazy `, but + it prioritizes resource savings over crawl speed. It is + functionally equivalent to running the spider multiple times in a + row, one per seed request. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS @@ -1969,7 +2037,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def yield_seeds(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1998,7 +2066,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def yield_seeds(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index b45b12540..782ac1e26 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -160,7 +160,7 @@ item_scraped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.yield_seeds`. :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped @@ -181,7 +181,7 @@ item_dropped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.yield_seeds`. :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a @@ -205,7 +205,7 @@ item_error :param response: the response being processed when the exception was raised, or ``None`` if it was yielded from - :meth:`~scrapy.Spider.start_requests`. + :meth:`~scrapy.Spider.yield_seeds`. :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b59cabe1..62d34c52e 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -74,6 +74,31 @@ object gives you access, for example, to the :ref:`settings `. .. class:: SpiderMiddleware + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + + .. method:: process_seeds(seeds: AsyncIterator[Any], /) -> AsyncIterator[Any] + :async: + + Iterate over the output of :meth:`~scrapy.Spider.yield_seeds` or that + of the :meth:`process_seeds` method of an earlier spider middleware, + overriding it. + + You may yield :class:`~scrapy.Request` or :ref:`item ` + objects, same as :meth:`~scrapy.Spider.yield_seeds`, from *seeds* or + not. + + As with :meth:`~scrapy.Spider.yield_seeds`, how this method is iterated + is controlled by :setting:`SEEDING_POLICY`. + .. method:: process_spider_input(response, spider) This method is called for each response that goes through the spider @@ -168,42 +193,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object - .. method:: process_start_requests(start_requests, spider) - - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). - - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. - - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). - - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` - - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-spider-middleware-ref: Built-in spider middleware reference diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e1b1c5ad6..88eed477a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -12,16 +12,16 @@ parsing pages for a particular site (or, in some cases, a group of sites). For spiders, the scraping cycle goes through something like this: -1. You start by generating the initial Requests to crawl the first URLs, and +1. You start by generating the initial requests to crawl the first URLs, and specify a callback function to be called with the response downloaded from those requests. - The first requests to perform are obtained by calling the - :meth:`~scrapy.Spider.start_requests` method which (by default) - generates :class:`~scrapy.Request` for the URLs specified in the - :attr:`~scrapy.Spider.start_urls` and the - :attr:`~scrapy.Spider.parse` method as callback function for the - Requests. + The first requests to perform are obtained by iterating the + :meth:`~scrapy.Spider.yield_seeds` method, which by default yields a + :class:`~scrapy.Request` object for each URL in the + :attr:`~scrapy.Spider.start_urls` spider attribute, with the + :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` + function to handle each :class:`~scrapy.http.Response`. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, @@ -48,14 +48,7 @@ scrapy.Spider ============= .. class:: scrapy.spiders.Spider -.. class:: scrapy.Spider() - - This is the simplest spider, and the one from which every other spider - must inherit (including spiders that come bundled with Scrapy, as well as spiders - that you write yourself). It doesn't provide any special functionality. It just - provides a default :meth:`start_requests` implementation which sends requests from - the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` - for each of the resulting responses. +.. autoclass:: scrapy.Spider .. attribute:: name @@ -81,12 +74,7 @@ scrapy.Spider Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. - .. attribute:: start_urls - - A list of URLs where the spider will begin to crawl from, when no - particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data - contained in the start URLs. + .. autoattribute:: start_urls .. attribute:: custom_settings @@ -149,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`start_requests` method, handlers of the + :meth:`yield_seeds` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -201,42 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. method:: start_requests() - - This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects - ` for - this spider. It is called by Scrapy when the spider is opened for - scraping. Scrapy calls it only once, so it is safe to implement - :meth:`start_requests` as a generator. - - The default implementation generates ``Request(url, dont_filter=True)`` - for each url in :attr:`start_urls`. - - If you want to change the Requests used to start scraping a domain, this is - the method to override. For example, if you need to start by logging in using - a POST request, you could do: - - .. code-block:: python - - import scrapy - - - class MySpider(scrapy.Spider): - name = "myspider" - - def start_requests(self): - return [ - scrapy.FormRequest( - "http://www.example.com/login", - formdata={"user": "john", "pass": "secret"}, - callback=self.logged_in, - ) - ] - - def logged_in(self, response): - # here you would extract links to follow and return Requests for - # each of them, with another callback - pass + .. automethod:: yield_seeds .. method:: parse(response) @@ -308,7 +261,7 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; +Instead of :attr:`~.start_urls` you can use :meth:`~.yield_seeds` directly; to give data more structure you can use :class:`~scrapy.Item` objects: .. skip: next @@ -322,7 +275,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - def start_requests(self): + async def yield_seeds(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -376,11 +329,11 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - def start_requests(self): + async def yield_seeds(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") -If you are :ref:`running Scrapy from a script `, you can -specify spider arguments when calling +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling :class:`CrawlerProcess.crawl ` or :class:`CrawlerRunner.crawl `: @@ -940,10 +893,11 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - def start_requests(self): - requests = list(super(MySpider, self).start_requests()) - requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls] - return requests + async def yield_seeds(self): + async for seed in super().yield_seeds(): + yield seed + for url in self.other_urls: + yield Request(url, self.parse_other) def parse_shop(self, response): pass # ... scrape shop here ... diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 119dfdabb..69a09d2b6 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,7 +34,7 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) - def start_requests(self): + async def yield_seeds(self): url = self.benchurl if self.latency is not None: url += f"?latency={self.latency}" diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 16dae6ac4..fccf7c2d3 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,9 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import Iterable - - from scrapy import Request + from collections.abc import AsyncIterator class Command(ScrapyCommand): @@ -61,10 +59,10 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self) -> Iterable[Request]: + async def yield_seeds(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" - return [scrapy.Request(url, dont_filter=True)] + yield scrapy.Request(url, dont_filter=True) def parse(self, response: Response) -> Any: assert isinstance(response, TextResponse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 1ce155da7..1982d25c5 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -80,10 +80,17 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader + async def yield_seeds(self): + requests = conman.from_spider(self, self._result) + for request in requests: + yield request + with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] + + spidercls._result = result # type: ignore[assignment,method-assign,return-value] + spidercls.yield_seeds = yield_seeds # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 8a8d04ff6..97eeeffa1 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -89,5 +89,12 @@ class Command(ScrapyCommand): spidercls = spider_loader.load(opts.spider) else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) + + async def yield_seeds(self): + yield self._request + + spidercls._request = request # type: ignore[assignment] + spidercls.yield_seeds = yield_seeds + + self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 61aea3ee4..71c9305d4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: import argparse - from collections.abc import AsyncGenerator, Coroutine, Iterable + from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable from twisted.python.failure import Failure @@ -258,11 +258,11 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider: Spider) -> Iterable[Request]: + async def yield_seeds(spider: Spider) -> AsyncIterator[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] + self.spidercls.yield_seeds = yield_seeds # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 61f444e31..42fbb7219 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -8,10 +8,10 @@ For more information see docs/topics/architecture.rst from __future__ import annotations import logging +from enum import Enum from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -20,12 +20,13 @@ from scrapy import signals from scrapy.core.scraper import Scraper, _HandleOutputDeferred from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import Callable, Generator, Iterable, Iterator + from collections.abc import AsyncIterator, Callable, Generator from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -44,18 +45,19 @@ _T = TypeVar("_T") class Slot: def __init__( self, - start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce[None], scheduler: BaseScheduler, + *, + seeds: AsyncIterator[Any] | None, ) -> None: self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() - self.start_requests: Iterator[Request] | None = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) + self.seeds: AsyncIterator[Any] | None = seeds def add_request(self, request: Request) -> None: self.inprogress.add(request) @@ -78,6 +80,13 @@ class Slot: self.closing.callback(None) +class _SeedingPolicy(Enum): + lazy = "lazy" + front_load = "front-load" + greedy = "greedy" + serial = "serial" + + class ExecutionEngine: def __init__( self, @@ -103,6 +112,19 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None + self._load_seeding_policy() + + def _load_seeding_policy(self) -> None: + try: + policy = _SeedingPolicy(self.settings["SEEDING_POLICY"]) + except ValueError: + supported_values = ", ".join(policy.value for policy in _SeedingPolicy) + raise ValueError( + f"The value of the SEEDING_POLICY setting " + f"({self.settings['SEEDING_POLICY']!r}) is not supported. " + f"Supported values: {supported_values}." + ) + self._feed = getattr(self, f"_{policy.name}_feed") def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -164,7 +186,8 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - def _next_request(self) -> None: + @inlineCallbacks + def _lazy_feed(self) -> Generator[Deferred[Any], Any, None]: if self.slot is None: return @@ -179,29 +202,23 @@ class ExecutionEngine: ): pass - if self.slot.start_requests is not None and not self._needs_backout(): + if self.slot.seeds is not None and not self._needs_backout(): try: - request_or_item = next(self.slot.start_requests) - except StopIteration: - self.slot.start_requests = None + request_or_item = yield deferred_from_coro(self.slot.seeds.__anext__()) + except StopAsyncIteration: + self.slot.seeds = None except Exception: - self.slot.start_requests = None + self.slot.seeds = None logger.error( - "Error while obtaining start requests", + "Error while reading seeds", exc_info=True, extra={"spider": self.spider}, ) else: if isinstance(request_or_item, Request): self.crawl(request_or_item) - elif is_item(request_or_item): - self.scraper.start_itemproc(request_or_item, response=None) else: - logger.error( - f"Got {request_or_item!r} among start requests. Only " - f"requests and items are supported. It will be " - f"ignored." - ) + self.scraper.start_itemproc(request_or_item, response=None) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() @@ -289,7 +306,7 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False - if self.slot.start_requests is not None: # not all start requests are handled + if self.slot.seeds is not None: # not all start requests are handled return False return not self.slot.scheduler.has_pending_requests() @@ -373,18 +390,15 @@ class ExecutionEngine: def open_spider( self, spider: Spider, - start_requests: Iterable[Request] = (), close_if_idle: bool = True, ) -> Generator[Deferred[Any], Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) - nextcall = CallLaterOnce(self._next_request) + nextcall = CallLaterOnce(self._feed) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - start_requests = yield self.scraper.spidermw.process_start_requests( - start_requests, spider - ) - self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) + seeds = yield self.scraper.spidermw.process_seeds(spider) + self.slot = Slot(close_if_idle, nextcall, scheduler, seeds=seeds) self.spider = spider if hasattr(scheduler, "open") and (d := scheduler.open(spider)): yield d diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 4bb143dfd..287b15c4b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -56,7 +56,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): The original sources of said requests are: - * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``yield_seeds`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 85a3b5895..7a6d9bafe 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,16 +7,17 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, Callable, Iterable -from inspect import isasyncgenfunction, iscoroutine +from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable +from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from warnings import warn from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen @@ -59,8 +60,8 @@ class SpiderMiddlewareManager(MiddlewareManager): super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft(mw.process_start_requests) + if hasattr(mw, "process_seeds"): + self.methods["process_seeds"].appendleft(mw.process_seeds) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) process_spider_exception = getattr(mw, "process_spider_exception", None) @@ -323,10 +324,65 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd2.addErrback(process_spider_exception) return dfd2 - def process_start_requests( - self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred[Iterable[Request]]: - return self._process_chain("process_start_requests", start_requests, spider) + @inlineCallbacks + def process_seeds(self, spider: Spider) -> Deferred[AsyncIterator[Any]]: + self._check_deprecated_start_requests_use(spider) + seeds = yield self._iter_seeds(spider) + seeds = yield self._process_chain("process_seeds", seeds) + return seeds + + @staticmethod + def _check_deprecated_start_requests_use(spider: Spider): + start_requests_cls = None + yield_seeds_cls = None + spidercls = spider.__class__ + mro = spidercls.__mro__ + + for cls in mro: + cls_dict = cls.__dict__ + if start_requests_cls is None and "start_requests" in cls_dict: + start_requests_cls = cls + if yield_seeds_cls is None and "yield_seeds" in cls_dict: + yield_seeds_cls = cls + if start_requests_cls is not None and yield_seeds_cls is not None: + break + + # Spider defines both, start_requests and yield_seeds. + assert start_requests_cls is not None + assert yield_seeds_cls is not None + + if ( + start_requests_cls is not Spider + and yield_seeds_cls is not start_requests_cls + and mro.index(start_requests_cls) < mro.index(yield_seeds_cls) + ): + src = global_object_name(start_requests_cls) + if start_requests_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + warn( + f"{src} defines the deprecated start_requests() method. " + f"start_requests() has been deprecated in favor of a new " + f"method, yield_seeds(), to support asynchronous code " + f"execution. start_requests() will stop being called in a " + f"future version of Scrapy. If you use Scrapy VERSION or " + f"higher only, replace start_requests() with yield_seeds(); " + f"note that yield_seeds() is a coroutine (async def). If you " + f"need to maintain compatibility with lower Scrapy versions, " + f"when overriding start_requests() in a spider class, " + f"override yield_seeds() as well; you can use super() to " + f"reuse the inherited yield_seeds() implementation without " + f"copy-pasting. See the release notes of Scrapy VERSION for " + f"details: https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + + @staticmethod + def _iter_seeds(spider: Spider): + fn = spider.yield_seeds + if isasyncgenfunction(fn): + return fn().__aiter__() + assert iscoroutinefunction(fn) + return deferred_from_coro(fn()) # This method is only needed until _async compatibility methods are removed. @staticmethod diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1ec1e31dc..88d618085 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -151,8 +151,7 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - start_requests = iter(self.spider.start_requests()) - yield self.engine.open_spider(self.spider, start_requests) + yield self.engine.open_spider(self.spider) yield maybeDeferred(self.engine.start) except Exception: self.crawling = False diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index e24f6874d..0d2b3aa99 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,11 +138,60 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") + + #: :class:`~collections.abc.Callable` to parse the + #: :class:`~scrapy.http.Response` to this request once received. + #: + #: The callable must expect the response as its first parameter, and + #: support any additional keyword arguments set through + #: :attr:`cb_kwargs`. + #: + #: In addition to an arbitrary callable, the following values are also + #: supported: + #: + #: - ``None`` (default), which indicates that the + #: :meth:`~scrapy.Spider.parse` method of the spider must be used. + #: + #: - :func:`~scrapy.http.request.NO_CALLBACK`. + #: + #: If an unhandled exception is raised during request or response + #: processing, i.e. by a :ref:`spider middleware + #: `, :ref:`downloader middleware + #: ` or download handler + #: (:setting:`DOWNLOAD_HANDLERS`), :attr:`errback` is called instead. + #: + #: .. tip:: + #: :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware` + #: raises exceptions for non-2xx responses by default, sending them + #: to the :attr:`errback` instead. + #: + #: .. seealso:: + #: :ref:`topics-request-response-ref-request-callback-arguments` self.callback: CallbackT | None = callback + + #: :class:`~collections.abc.Callable` to handle exceptions raised + #: during request or response processing. + #: + #: The callable must expect a :exc:`~twisted.python.failure.Failure` as + #: its first parameter. + #: + #: .. seealso:: :ref:`topics-request-response-ref-errbacks` self.errback: Callable[[Failure], Any] | None = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) + + #: Whether this request may be filtered out by :ref:`components + #: ` that support filtering out requests (``False``, + #: default), or those components should not filter out this request + #: (``True``). + #: + #: This attribute is commonly set to ``True`` to prevent duplicate + #: requests from being filtered out. + #: + #: When defining the start URLs of a spider through + #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by + #: default. See :meth:`~scrapy.Spider.yield_seeds`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index f10e91beb..6315b7adc 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -98,7 +98,7 @@ class LogFormatter: """Logs a message when an item is scraped by a spider.""" src: Any if response is None: - src = f"{global_object_name(spider.__class__)}.start_requests" + src = f"{global_object_name(spider.__class__)}.yield_seeds" elif isinstance(response, Failure): src = response.getErrorMessage() else: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c473b369c..0464e22be 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -308,6 +308,8 @@ SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SEEDING_POLICY = "lazy" + SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e255e91cc..b8d360d81 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,7 +15,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import AsyncIterator, Iterable from twisted.internet.defer import Deferred @@ -29,21 +29,25 @@ if TYPE_CHECKING: class Spider(object_ref): - """Base class for scrapy spiders. All spiders must inherit from this - class. + """Base class that any spider must subclass. + + It provides a default :meth:`yield_seeds` implementation that sends + requests based on the :attr:`start_urls` class attribute and calls the + :meth:`parse` method for each response. """ name: str custom_settings: dict[_SettingsKeyT, Any] | None = None + #: Seed URLs. See :meth:`yield_seeds`. + start_urls: list[str] = [] + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) - if not hasattr(self, "start_urls"): - self.start_urls: list[str] = [] @property def logger(self) -> SpiderLoggerAdapter: @@ -72,7 +76,48 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self) -> Iterable[Request]: + async def yield_seeds(self) -> AsyncIterator[Any]: + """Yield the initial :class:`~scrapy.Request` objects to send. + + .. versionadded:: VERSION + + For example: + + .. code-block:: python + + from scrapy import Request, Spider + + + class MySpider(Spider): + name = "myspider" + + async def yield_seeds(self): + yield Request("https://toscrape.com/") + + The default implementation reads URLs from :attr:`start_urls` and + yields a request for each with :attr:`~scrapy.Request.dont_filter` + enabled. It is functionally equivalent to: + + .. code-block:: python + + async def yield_seeds(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + + You can also yield :ref:`items `. For example: + + .. code-block:: python + + async def yield_seeds(self): + yield {"foo": "bar"} + + Use :setting:`SEEDING_POLICY` to set how :meth:`yield_seeds` is + iterated. + """ + for seed in self.start_requests(): + yield seed + + def start_requests(self) -> Iterable[Any]: if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 4ec2919f7..770ec495c 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -14,6 +14,10 @@ if TYPE_CHECKING: class InitSpider(Spider): """Base Spider with initialization facilities""" + async def yield_seeds(self) -> AsyncIterator[Any]: + async for seed in super().yield_seeds(): + yield seed + def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() return cast(Iterable[Request], iterate_spider_output(self.init_request())) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 39033ac3c..291f5ee25 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,6 +53,10 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + async def yield_seeds(self) -> AsyncIterator[Any]: + async for seed in super().yield_seeds(): + yield seed + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8c9a86dce..ea3276c4b 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,14 +43,13 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. + async def process_seeds(self, seeds): + # Called with the seeds from the spider yield_seeds() method or with + # the output of the maching method of an earlier spider middleware. # Must return only requests (not items). - for r in start_requests: - yield r + async for seed in seeds: + yield seed def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index a46a8806b..686c35162 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,7 +13,7 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def yield_seeds(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") return [] diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 5e72aa6d4..e814a28d9 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6df6d76fa..a8222b01f 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f013eed27..1e68dd016 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -14,7 +14,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index e9d6d8875..09d81593d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -15,7 +15,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index c72a0a17c..8d71efccd 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -16,7 +16,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 7b0497bde..26343da6a 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - def start_requests(self): + async def yield_seeds(self): yield scrapy.Request(self.url) def parse(self, response): diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 9f7eaf2ae..a3ca7b5bf 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 078cb72cb..50ea417fb 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -7,7 +7,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index eee808c32..67125247f 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -8,7 +8,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 814a2a46d..702266745 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -9,7 +9,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 38ca4c4f1..d1079b1a3 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,7 +17,7 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index b397608ec..4f00514c7 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,7 +9,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 2d8769501..a61cadd06 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index de76e13e8..67c56cbc8 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -10,7 +10,7 @@ class NoRequestsSpider(Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 2f1bb7713..2643e6823 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -26,7 +26,7 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - def start_requests(self): + async def yield_seeds(self): yield Request(self.url) def parse(self, response): diff --git a/tests/spiders.py b/tests/spiders.py index da923de6e..5075a795b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -68,7 +68,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - def start_requests(self): + async def yield_seeds(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -105,7 +105,7 @@ class LogSpider(MetaSpider): class SlowSpider(DelaySpider): name = "slow" - def start_requests(self): + async def yield_seeds(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -255,7 +255,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - def start_requests(self): + async def yield_seeds(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenStartRequestsSpider(FollowAllSpider): +class BrokenYieldSeedsSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -327,7 +327,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - def start_requests(self): + async def yield_seeds(self): if self.fail_before_yield: 1 / 0 @@ -338,22 +338,20 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, ( - "All start requests consumed before any download happened" - ) + assert self.seedsseen, "All seeds consumed before any download happened" def parse(self, response): self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) -class StartRequestsItemSpider(FollowAllSpider): - def start_requests(self): +class YieldSeedsItemSpider(FollowAllSpider): + async def yield_seeds(self): yield {"name": "test item"} -class StartRequestsGoodAndBadOutput(FollowAllSpider): - def start_requests(self): +class YieldSeedsGoodAndBadOutput(FollowAllSpider): + async def yield_seeds(self): yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -365,7 +363,7 @@ class SingleRequestSpider(MetaSpider): callback_func = None errback_func = None - def start_requests(self): + async def yield_seeds(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -386,13 +384,13 @@ class SingleRequestSpider(MetaSpider): return None -class DuplicateStartRequestsSpider(MockServerSpider): +class DuplicateYieldSeedsSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 dupe_factor = 3 - def start_requests(self): + async def yield_seeds(self): for i in range(self.distinct_urls): for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") @@ -417,7 +415,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - def start_requests(self): + async def yield_seeds(self): test_body = b""" Page title<title></head> @@ -471,7 +469,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - def start_requests(self): + async def yield_seeds(self): test_body = b""" <html> <head><title>Page title<title></head> @@ -516,7 +514,7 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - def start_requests(self): + async def yield_seeds(self): body = b"a" * self.full_response_length url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=body, errback=self.errback) @@ -545,7 +543,7 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - def start_requests(self): + async def yield_seeds(self): yield Request(self.mockserver.url("/status"), errback=self.errback) def parse(self, response): diff --git a/tests/test_commands.py b/tests/test_commands.py index f63e05628..874ecf93c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -670,7 +670,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug("It Works!") return [] """ @@ -680,7 +680,7 @@ import scrapy class BadSpider(scrapy.Spider): name = "bad" - def start_requests(self): + async def yield_seeds(self): raise Exception("oops!") """ @@ -771,9 +771,9 @@ class MySpider(scrapy.Spider): log = self.get_log("", name="myspider.txt") assert "Unable to load" in log - def test_start_requests_errors(self): + def test_yield_seeds_errors(self): log = self.get_log(self.badspider, name="badspider.py") - assert "start_requests" in log + assert "yield_seeds" in log assert "badspider.py" in log def test_asyncio_enabled_true(self): @@ -846,7 +846,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return [] """ @@ -862,7 +862,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) @@ -888,7 +888,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): return [] """ args = ["-o", "example1.json", "-O", "example2.json"] @@ -904,7 +904,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return [] """ @@ -983,7 +983,7 @@ class MySpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def yield_seeds(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") return [] """ @@ -1001,9 +1001,9 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): raise unittest.SkipTest("Windows required for .pyw files") return super().setUp() - def test_start_requests_errors(self): + def test_yield_seeds_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - assert "start_requests" in log + assert "yield_seeds" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): @@ -1053,7 +1053,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug('It works!') return [] """ @@ -1067,7 +1067,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return [] """ @@ -1083,7 +1083,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) @@ -1109,7 +1109,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def yield_seeds(self): return [] """ args = ["-o", "example1.json", "-O", "example2.json"] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb961ace2..d06e186cf 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -511,8 +511,9 @@ class TestContractsManager(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(self_): # pylint: disable=no-self-argument - return self.conman.from_spider(self_, self.results) + async def yield_seeds(self_): # pylint: disable=no-self-argument + for seed in self.conman.from_spider(self_, self.results): + yield seed def parse_first(self, response): self.visited += 1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5766f9313..b5472063d 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -35,7 +34,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenStartRequestsSpider, + BrokenYieldSeedsSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -44,14 +43,14 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateStartRequestsSpider, + DuplicateYieldSeedsSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, - StartRequestsGoodAndBadOutput, - StartRequestsItemSpider, + YieldSeedsGoodAndBadOutput, + YieldSeedsItemSpider, ) @@ -164,9 +163,9 @@ class TestCrawl(TestCase): self._assert_retried(log) @defer.inlineCallbacks - def test_start_requests_bug_before_yield(self): + def test_yield_seeds_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenYieldSeedsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -175,9 +174,9 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_bug_yielding(self): + def test_yield_seeds_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenYieldSeedsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -186,52 +185,44 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_items(self): + def test_yield_seeds_items(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsItemSpider) + crawler = get_crawler(YieldSeedsItemSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 @defer.inlineCallbacks - def test_start_requests_unsupported_output(self): + def test_yield_seeds_unsupported_output(self): + """Anything that is not a request or a seeding policy is assumed to be + an item, avoiding a potentially expensive call to itemadapter.is_item, + and letting instead things fail when ItemAdapter is actually used on + the corresponding non-item object.""" with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler = get_crawler(YieldSeedsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 2 - assert log.records[0].msg == ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ) - assert re.match( - ( - r"^Got <object object at 0x[0-9a-fA-F]+> among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert len(log.records) == 0 @defer.inlineCallbacks - def test_start_requests_laziness(self): + def test_yield_seeds_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenStartRequestsSpider, settings) + crawler = get_crawler(BrokenYieldSeedsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( 99 ), crawler.spider.seedsseen @defer.inlineCallbacks - def test_start_requests_dupes(self): + def test_yield_seeds_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateYieldSeedsSpider, settings) yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) assert crawler.spider.visited == 6 - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateYieldSeedsSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, @@ -313,10 +304,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # start requests doesn't set Referer header + # test_yield_seeds doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to start request url + # following request sets Referer to test_yield_seeds url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header @@ -375,15 +366,15 @@ with multiples lines Test whether errors happening anywhere in Crawler.crawl() are properly reported (and not somehow swallowed) after a graceful engine shutdown. The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.start_requests(), - SpiderMiddleware.process_start_requests(), etc. + spiders/middlewares/etc., e.g. raised in Spider.test_yield_seeds(), + SpiderMiddleware.process_test_yield_seeds(), etc. """ class TestError(Exception): pass class FaultySpider(SimpleSpider): - def start_requests(self): + async def yield_seeds(self): raise TestError crawler = get_crawler(FaultySpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0bbcc0843..caf7cb41d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -147,7 +147,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def yield_seeds(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -227,7 +227,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def yield_seeds(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -307,7 +307,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def yield_seeds(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -387,7 +387,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def yield_seeds(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -574,7 +574,7 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): + async def yield_seeds(self): return [] diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8e718ad5b..5d1161750 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -24,7 +24,7 @@ class TestManagerBase(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) self.crawler.engine = self.crawler._create_engine() - return self.crawler.engine.open_spider(self.spider, start_requests=()) + return self.crawler.engine.open_spider(self.spider) def tearDown(self): return self.crawler.engine.close_spider(self.spider) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 15b3ad5af..d9685d93b 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -28,7 +28,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } - def start_requests(self): + async def yield_seeds(self): self.times = {None: []} slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bac8d273..9ca241675 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -92,8 +92,9 @@ class MySpider(Spider): class DupeFilterSpider(MySpider): - def start_requests(self): - return (Request(url) for url in self.start_urls) # no dont_filter=True + async def yield_seeds(self): + for url in self.start_urls: + yield Request(url) # no dont_filter=True class DictItemsSpider(MySpider): @@ -490,7 +491,12 @@ def test_request_scheduled_signal(caplog): engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() - engine.slot = Slot((), None, Mock(), scheduler) + + async def seeds(): + return + yield + + engine.slot = Slot(None, Mock(), scheduler, seeds=seeds) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") engine._schedule_request(keep_request, spider) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 0ae86235c..4d6da61db 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -69,7 +69,7 @@ class AsyncDefNotAsyncioPipeline: class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def yield_seeds(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a21cb43ff..60aadf2f0 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_start_requests(self, start_requests, spider): - for request in start_requests: + def process_test_yield_seeds(self, test_yield_seeds, spider): + for request in test_yield_seeds: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_start_requests"] = True + request.cb_kwargs["from_process_test_yield_seeds"] = True yield request def process_spider_input(self, response, spider): @@ -62,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] - def start_requests(self): + async def yield_seeds(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -139,10 +139,10 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 2) def parse_spider_mw( - self, response, from_process_spider_input, from_process_start_requests + self, response, from_process_spider_input, from_process_test_yield_seeds ): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_start_requests)) + self.checks.append(bool(from_process_test_yield_seeds)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_signals.py b/tests/test_signals.py index a508eb41a..7985bc520 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -10,7 +10,7 @@ from tests.mockserver import MockServer class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def yield_seeds(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} diff --git a/tests/test_spider.py b/tests/test_spider.py index af29872a8..05ac8ff80 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,7 @@ import gzip -import inspect import warnings from io import BytesIO -from logging import WARNING +from logging import ERROR, WARNING from pathlib import Path from typing import Any from unittest import mock @@ -45,12 +44,6 @@ class SpiderTest(unittest.TestCase): self.assertEqual(spider.name, "example.com") self.assertEqual(spider.start_urls, []) - def test_start_requests(self): - spider = self.spider_class("example.com") - start_requests = spider.start_requests() - self.assertTrue(inspect.isgenerator(start_requests)) - self.assertEqual(list(start_requests), []) - def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") @@ -475,12 +468,22 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, "_follow_links")) self.assertFalse(spider._follow_links) + @inlineCallbacks def test_start_url(self): - spider = self.spider_class("example.com") - spider.start_url = "https://www.example.com" + class TestSpider(self.spider_class): + name = "test" + start_url = "https://www.example.com" - with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"): - list(spider.start_requests()) + crawler = Crawler(TestSpider) + with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: + yield crawler.crawl() + log.check( + ( + "scrapy.core.engine", + "ERROR", + "Error while reading seeds", + ), + ) class SitemapSpiderTest(SpiderTest): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a9f3876bb..cb926a0fe 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,11 +1,13 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from inspect import isasyncgen from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -110,7 +112,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_start_requests. + Should work for process_spider_output and, when it's supported, process_test_yield_seeds. """ ITEM_TYPE: type | tuple @@ -319,37 +321,44 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): ) -class ProcessStartRequestsSimpleMiddleware: - def process_start_requests(self, start_requests, spider): - yield from start_requests +class ProcessYieldSeedsSimpleMiddleware: + def process_test_yield_seeds(self, test_yield_seeds, spider): + yield from test_yield_seeds -class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): - """process_start_requests tests for simple start_requests""" +class ProcessYieldSeedsSimple(BaseAsyncSpiderMiddlewareTestCase): + """process_test_yield_seeds tests for simple test_yield_seeds""" ITEM_TYPE = (Request, dict) - MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware - def _start_requests(self): - for i in range(2): - yield Request(f"https://example.com/{i}", dont_filter=True) - yield {"name": "test item"} + @inlineCallbacks + def _get_processed_seeds(self, *mw_classes): + class TestSpider(Spider): + name = "test" - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): - setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + async def yield_seeds(self): + for i in range(2): + yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} + + setting = self._construct_mw_setting(*mw_classes) self.crawler = get_crawler( - Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) - self.spider = self.crawler._create_spider("foo") + self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - start_requests = iter(self._start_requests()) - results = yield self.mwman.process_start_requests(start_requests, self.spider) + results = yield self.mwman.process_seeds(self.spider) return results + @inlineCallbacks def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + seeds = yield self._get_processed_seeds(self.MW_SIMPLE) + self.assertTrue(isasyncgen(seeds)) + seed_list = yield deferred_from_coro(collect_asyncgen(seeds)) + self.assertEqual(len(seed_list), self.RESULT_COUNT) + self.assertIsInstance(seed_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index f9eb93d6b..47ffd94ac 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -31,7 +31,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - def start_requests(self): + async def yield_seeds(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index e51957497..72d399687 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -36,7 +36,7 @@ class RecoverySpider(Spider): }, } - def start_requests(self): + async def yield_seeds(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -73,7 +73,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - def start_requests(self): + async def yield_seeds(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -83,7 +83,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - def start_requests(self): + async def yield_seeds(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -103,7 +103,7 @@ class GeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def yield_seeds(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -140,7 +140,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def yield_seeds(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -215,7 +215,7 @@ class GeneratorOutputChainSpider(Spider): }, } - def start_requests(self): + async def yield_seeds(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -287,8 +287,8 @@ class NotGeneratorOutputChainSpider(Spider): }, } - def start_requests(self): - return [Request(self.mockserver.url("/status?n=200"))] + async def yield_seeds(self): + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): return [