From 036f3e562716aaf67a4d0ff1c8011281394ef240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 May 2025 19:04:03 +0200 Subject: [PATCH] Support asynchronous start requests (#6729) --- docs/faq.rst | 24 +- docs/intro/tutorial.rst | 39 +- docs/news.rst | 133 ++++++- docs/topics/api.rst | 6 + docs/topics/architecture.rst | 2 +- docs/topics/components.rst | 4 + docs/topics/coroutines.rst | 28 +- docs/topics/jobs.rst | 6 +- docs/topics/request-response.rst | 10 +- docs/topics/scheduler.rst | 8 +- docs/topics/settings.rst | 62 ++- docs/topics/signals.rst | 19 +- docs/topics/spider-middleware.rst | 46 ++- docs/topics/spiders.rst | 119 +++--- docs/topics/telnetconsole.rst | 12 +- extras/qpsclient.py | 4 + pyproject.toml | 4 +- scrapy/commands/bench.py | 8 +- scrapy/commands/check.py | 10 +- scrapy/commands/fetch.py | 8 +- scrapy/commands/parse.py | 6 +- scrapy/commands/shell.py | 4 +- scrapy/core/engine.py | 248 +++++++----- scrapy/core/scheduler.py | 287 ++++++++++---- scrapy/core/scraper.py | 333 ++++++++-------- scrapy/core/spidermw.py | 203 ++++++++-- scrapy/crawler.py | 8 +- scrapy/extensions/telnet.py | 1 - scrapy/http/request/__init__.py | 12 +- scrapy/logformatter.py | 2 +- scrapy/pqueues.py | 126 ++++-- scrapy/settings/default_settings.py | 3 + scrapy/shell.py | 23 +- scrapy/signalmanager.py | 21 +- scrapy/signals.py | 1 + scrapy/spidermiddlewares/base.py | 63 +-- scrapy/spidermiddlewares/depth.py | 11 +- scrapy/spidermiddlewares/offsite.py | 5 +- scrapy/spidermiddlewares/referer.py | 5 +- scrapy/spidermiddlewares/start.py | 31 ++ scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/__init__.py | 79 +++- scrapy/spiders/crawl.py | 6 +- scrapy/spiders/init.py | 10 +- scrapy/spiders/sitemap.py | 6 +- .../project/module/middlewares.py.tmpl | 13 +- scrapy/utils/asyncgen.py | 8 +- scrapy/utils/defer.py | 34 +- scrapy/utils/engine.py | 8 +- scrapy/utils/python.py | 17 +- scrapy/utils/reactor.py | 25 +- sep/sep-018.rst | 2 +- tests/CrawlerProcess/args_settings.py | 5 +- tests/CrawlerProcess/asyncio_custom_loop.py | 5 +- .../asyncio_enabled_no_reactor.py | 5 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 5 +- .../asyncio_enabled_reactor_different_loop.py | 5 +- .../asyncio_enabled_reactor_same_loop.py | 5 +- .../caching_hostname_resolver.py | 2 +- tests/CrawlerProcess/multi.py | 5 +- tests/CrawlerProcess/reactor_default.py | 5 +- .../reactor_default_twisted_reactor_select.py | 5 +- tests/CrawlerProcess/reactor_select.py | 5 +- ..._select_subclass_twisted_reactor_select.py | 5 +- .../reactor_select_twisted_reactor_select.py | 5 +- tests/CrawlerProcess/simple.py | 5 +- tests/CrawlerRunner/change_reactor.py | 5 +- tests/CrawlerRunner/ip_address.py | 2 +- tests/__init__.py | 6 + tests/spiders.py | 36 +- .../__init__.py | 17 +- tests/test_commands.py | 63 +-- tests/test_contracts.py | 5 +- tests/test_crawl.py | 70 +--- tests/test_crawler.py | 13 +- tests/test_downloadermiddleware.py | 2 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine.py | 39 +- tests/test_engine_loop.py | 364 ++++++++++++++++++ tests/test_pipelines.py | 2 +- tests/test_request_cb_kwargs.py | 14 +- tests/test_scheduler.py | 35 +- tests/test_signals.py | 21 +- tests/test_spider.py | 57 ++- tests/test_spider_start.py | 186 +++++++++ tests/test_spidermiddleware.py | 53 +-- tests/test_spidermiddleware_base.py | 74 ++-- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 16 +- tests/test_spidermiddleware_process_start.py | 352 +++++++++++++++++ tests/test_spidermiddleware_start.py | 44 +++ tests/utils/__init__.py | 9 + tox.ini | 2 +- 93 files changed, 2777 insertions(+), 936 deletions(-) create mode 100644 scrapy/spidermiddlewares/start.py create mode 100644 tests/test_engine_loop.py create mode 100644 tests/test_spider_start.py create mode 100644 tests/test_spidermiddleware_process_start.py create mode 100644 tests/test_spidermiddleware_start.py diff --git a/docs/faq.rst b/docs/faq.rst index da255f29e..1d09a0e63 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -96,30 +96,13 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. + .. _faq-bfo-dfo: Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- -By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which -basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. - -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: - -.. code-block:: python - - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" - -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +:ref:`DFO by default, but other orders are possible `. My Scrapy crawler has memory leaks. What can I do? @@ -436,6 +419,3 @@ See :issue:`2680`. .. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ .. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) -.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5041b49ea..c4e04364b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -164,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as an argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``start`` method +---------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.start` to create the initial requests for your +spider. .. code-block:: python @@ -794,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 9f476ee21..64a3ad2b1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,11 @@ Release notes Scrapy VERSION (unreleased) --------------------------- +Highlights: + +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` + (async) and changed how it is iterated. + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -20,6 +25,116 @@ Backward-incompatible changes :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + To reproduce the previous behavior, see :ref:`start-requests-lazy`. + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware ` no longer stops the + crawl. + +- In ``scrapy.core.engine.ExecutionEngine``: + + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). + + - The ``slot`` attribute has been renamed to ``_slot`` and should not be + used. + +- In ``scrapy.core.engine``, the ``Slot`` class has been renamed to ``_Slot`` + and should not be used. + +- The ``slot`` :ref:`telnet variable ` has been removed. + +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler ` instance has been created. + +- When using :setting:`JOBDIR`, :ref:`start requests ` are + now serialized into their own, ``s``-suffixed priority folders. You can set + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` or ``""`` to change that, + but the side effects may be undesirable. See + :setting:`SCHEDULER_START_DISK_QUEUE` for details. + +Deprecations +~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``process_start_requests()`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``__init__`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) should now support a keyword-only + ``start_queue_cls`` parameter. + + (:issue:`6752`) + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider + middleware method, both :term:`asynchronous generators `. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without workarounds. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- Start requests are now :ref:`scheduled ` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- :class:`Crawler.signals ` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + +- Added a new :signal:`scheduler_empty` signal. + +- Added new settings: :setting:`SCHEDULER_START_DISK_QUEUE` and + :setting:`SCHEDULER_START_MEMORY_QUEUE`. + +- Added :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware`, which + sets :reqmeta:`is_start_request` to ``True`` on :ref:`start requests + `. + +- Exposed a new method of :class:`Crawler.engine + `: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + +Bug fixes +~~~~~~~~~ + +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. + + (:issue:`6729`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) @@ -29,7 +144,7 @@ Highlights: - Dropped support for Python 3.8, added support for Python 3.13 -- :meth:`~scrapy.Spider.start_requests` can now yield items +- ``scrapy.Spider.start_requests()`` can now yield items - Added :class:`~scrapy.http.JsonResponse` @@ -320,9 +435,13 @@ Deprecations New features ~~~~~~~~~~~~ -- :meth:`~scrapy.Spider.start_requests` can now yield items. +- ``scrapy.Spider.start_requests()`` can now yield items. (:issue:`5289`, :issue:`6417`) + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. + - Added a new :class:`~scrapy.http.Response` subclass, :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type `_. @@ -812,7 +931,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests` or in a handler of the + ``scrapy.Spider.start_requests()`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now @@ -3388,7 +3507,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -6309,7 +6428,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -6502,7 +6621,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -6565,7 +6684,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 5a00fd570..8e8f3a0c9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -280,3 +280,9 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4e53b6e3d..e8c510ea5 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 3a7644379..56f8c6498 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -37,6 +37,10 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` + +- :setting:`SCHEDULER_START_MEMORY_QUEUE` + - :setting:`SPIDER_MIDDLEWARES` Third-party Scrapy components may also let you define additional Scrapy diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 8af4ce71d..448bf07e7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -6,8 +6,8 @@ Coroutines .. versionadded:: 2.0 -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. +Scrapy :ref:`supports ` the :ref:`coroutine syntax ` +(i.e. ``async def``). .. _coroutine-support: @@ -18,6 +18,11 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): +- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded: VERSION + - :class:`~scrapy.Request` callbacks. If you are using any custom or third-party :ref:`spider middleware @@ -38,20 +43,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method of :ref:`spider middlewares `. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. + If defined as a coroutine, it must be an :term:`asynchronous generator`. + The input ``result`` parameter is an :term:`asynchronous iterable`. See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. .. versionadded:: 2.7 +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method + of :ref:`spider middlewares `, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded:: VERSION + +- :ref:`Signal handlers that support deferreds `. + .. _coroutine-deferred-apis: @@ -232,8 +243,9 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 0e705dc64..50bcaa6d6 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -46,9 +46,9 @@ Keeping persistent state between batches Sometimes you'll want to keep some persistent spider state between pause/resume batches. You can use the ``spider.state`` attribute for that, which should be a -dict. There's :ref:`a built-in extension ` that takes care of serializing, storing and -loading that attribute from the job directory, when the spider starts and -stops. +dict. There's :ref:`a built-in extension ` +that takes care of serializing, storing and loading that attribute from the job +directory, when the spider starts and stops. Here's an example of a callback that uses the spider state (other spider code is omitted for brevity): diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 77837378e..6ca0973d8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -127,10 +127,7 @@ Request objects body to bytes (if given as a string). :type encoding: str - :param priority: the priority of this request (defaults to ``0``). - The priority is used by the scheduler to define the order used to process - requests. Requests with a higher priority value will execute earlier. - Negative values are allowed in order to indicate relatively low-priority. + :param priority: sets :attr:`priority`, defaults to ``0``. :type priority: int :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. @@ -179,6 +176,8 @@ Request objects .. autoattribute:: errback + .. autoattribute:: priority + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents @@ -353,7 +352,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - def start_requests(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, @@ -647,6 +646,7 @@ Those are: * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` +* :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` * :reqmeta:`redirect_reasons` diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst index 57c24b76a..b6e54ebd7 100644 --- a/docs/topics/scheduler.rst +++ b/docs/topics/scheduler.rst @@ -26,9 +26,9 @@ Minimal scheduler interface :members: -Default Scrapy scheduler -======================== +Default scheduler +================= -.. autoclass:: Scheduler +.. autoclass:: Scheduler() :members: - :special-members: __len__ + :special-members: __init__, __len__ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3a61306d6..537e51e40 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1326,6 +1326,7 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. + .. setting:: JOBDIR JOBDIR @@ -1336,6 +1337,7 @@ Default: ``None`` A string indicating the directory for storing the state of a crawl when :ref:`pausing and resuming crawls `. + .. setting:: LOG_ENABLED LOG_ENABLED @@ -1700,23 +1702,28 @@ SCHEDULER_DISK_QUEUE Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` -Type of disk queue that will be used by scheduler. Other available types are -``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +Type of disk queue that will be used by the scheduler. Other available types +are ``scrapy.squeues.PickleFifoDiskQueue``, +``scrapy.squeues.MarshalFifoDiskQueue``, ``scrapy.squeues.MarshalLifoDiskQueue``. + .. setting:: SCHEDULER_MEMORY_QUEUE SCHEDULER_MEMORY_QUEUE ---------------------- + Default: ``'scrapy.squeues.LifoMemoryQueue'`` -Type of in-memory queue used by scheduler. Other available type is: +Type of in-memory queue used by the scheduler. Other available type is: ``scrapy.squeues.FifoMemoryQueue``. + .. setting:: SCHEDULER_PRIORITY_QUEUE SCHEDULER_PRIORITY_QUEUE ------------------------ + Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by the scheduler. Another available type is @@ -1726,6 +1733,51 @@ Type of priority queue used by the scheduler. Another available type is domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. + +.. setting:: SCHEDULER_START_DISK_QUEUE + +SCHEDULER_START_DISK_QUEUE +-------------------------- + +Default: ``'scrapy.squeues.PickleFifoDiskQueue'`` + +Type of disk queue (see :setting:`JOBDIR`) that the :ref:`scheduler +` uses for :ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_DISK_QUEUE`. + +.. queue-common-starts + +Use ``None`` or ``""`` to disable these separate queues entirely, and instead +have start requests share the same queues as other requests. + +.. note:: + + Disabling separate start request queues makes :ref:`start request order + ` unintuitive: start requests will be sent in order + only until :setting:`CONCURRENT_REQUESTS` is reached, then remaining start + requests will be sent in reverse order. + +.. queue-common-ends + + +.. setting:: SCHEDULER_START_MEMORY_QUEUE + +SCHEDULER_START_MEMORY_QUEUE +---------------------------- + +Default: ``'scrapy.squeues.FifoMemoryQueue'`` + +Type of in-memory queue that the :ref:`scheduler ` uses for +:ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_MEMORY_QUEUE`. + +.. include:: settings.rst + :start-after: queue-common-starts + :end-before: queue-common-ends + + .. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE SCRAPER_SLOT_MAX_ACTIVE_SIZE @@ -1957,7 +2009,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1986,7 +2038,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index b45b12540..66cb87fc5 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -131,6 +131,19 @@ engine_stopped This signal supports returning deferreds from its handlers. +scheduler_empty +~~~~~~~~~~~~~~~ + +.. signal:: scheduler_empty +.. function:: scheduler_empty() + + Sent whenever the engine asks for a pending request from the + :ref:`scheduler ` (i.e. calls its + :meth:`~scrapy.core.scheduler.BaseScheduler.next_request` method) and the + scheduler returns none. + + See :ref:`start-requests-lazy` for an example. + Item signals ------------ @@ -160,7 +173,7 @@ item_scraped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped @@ -181,7 +194,7 @@ item_dropped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a @@ -205,7 +218,7 @@ item_error :param response: the response being processed when the exception was raised, or ``None`` if it was yielded from - :meth:`~scrapy.Spider.start_requests`. + :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2211a822f..638035e64 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,30 +70,29 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_start_requests(start_requests, spider) + .. method:: process_start(start: AsyncIterator[Any], /) -> AsyncIterator[Any] + :async: - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). + Iterate over the output of :meth:`~scrapy.Spider.start` or that + of the :meth:`process_start` method of an earlier spider middleware, + overriding it. For example: - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. + .. code-block:: python - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` + You may yield the same type of objects as :meth:`~scrapy.Spider.start`. - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object + To write spider middlewares that work on Scrapy versions lower than + VERSION, define also a synchronous ``process_start_requests()`` method + that returns an iterable. For example: + + .. code-block:: python + + def process_start_requests(self, start, spider): + yield from start .. method:: process_spider_input(response, spider) @@ -154,6 +153,7 @@ one or more of these methods: :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output_async(response, result, spider) + :async: .. versionadded:: 2.7 @@ -417,6 +417,14 @@ String value Class name (as a string) .. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url +StartSpiderMiddleware +--------------------- + +.. module:: scrapy.spidermiddlewares.start + +.. autoclass:: StartSpiderMiddleware + + UrlLengthMiddleware ------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0a67240d6..891c4da05 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -12,16 +12,16 @@ parsing pages for a particular site (or, in some cases, a group of sites). For spiders, the scraping cycle goes through something like this: -1. You start by generating the initial Requests to crawl the first URLs, and +1. You start by generating the initial requests to crawl the first URLs, and specify a callback function to be called with the response downloaded from those requests. - The first requests to perform are obtained by calling the - :meth:`~scrapy.Spider.start_requests` method which (by default) - generates :class:`~scrapy.Request` for the URLs specified in the - :attr:`~scrapy.Spider.start_urls` and the - :attr:`~scrapy.Spider.parse` method as callback function for the - Requests. + The first requests to perform are obtained by iterating the + :meth:`~scrapy.Spider.start` method, which by default yields a + :class:`~scrapy.Request` object for each URL in the + :attr:`~scrapy.Spider.start_urls` spider attribute, with the + :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` + function to handle each :class:`~scrapy.http.Response`. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, @@ -48,14 +48,7 @@ scrapy.Spider ============= .. class:: scrapy.spiders.Spider -.. class:: scrapy.Spider() - - This is the simplest spider, and the one from which every other spider - must inherit (including spiders that come bundled with Scrapy, as well as spiders - that you write yourself). It doesn't provide any special functionality. It just - provides a default :meth:`start_requests` implementation which sends requests from - the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` - for each of the resulting responses. +.. autoclass:: scrapy.Spider .. attribute:: name @@ -81,12 +74,7 @@ scrapy.Spider Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. - .. attribute:: start_urls - - A list of URLs where the spider will begin to crawl from, when no - particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data - contained in the start URLs. + .. autoattribute:: start_urls .. attribute:: custom_settings @@ -149,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`start_requests` method, handlers of the + :meth:`start` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -201,42 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. method:: start_requests() - - This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects - ` for - this spider. It is called by Scrapy when the spider is opened for - scraping. Scrapy calls it only once, so it is safe to implement - :meth:`start_requests` as a generator. - - The default implementation generates ``Request(url, dont_filter=True)`` - for each url in :attr:`start_urls`. - - If you want to change the Requests used to start scraping a domain, this is - the method to override. For example, if you need to start by logging in using - a POST request, you could do: - - .. code-block:: python - - import scrapy - - - class MySpider(scrapy.Spider): - name = "myspider" - - def start_requests(self): - return [ - scrapy.FormRequest( - "http://www.example.com/login", - formdata={"user": "john", "pass": "secret"}, - callback=self.logged_in, - ) - ] - - def logged_in(self, response): - # here you would extract links to follow and return Requests for - # each of them, with another callback - pass + .. automethod:: start .. method:: parse(response) @@ -308,8 +261,9 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.Item` objects: +Instead of :attr:`~.start_urls` you can use :meth:`~scrapy.Spider.start` +directly; to give data more structure you can use :class:`~scrapy.Item` +objects: .. skip: next .. code-block:: python @@ -322,7 +276,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - def start_requests(self): + async def start(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -376,7 +330,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - def start_requests(self): + async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") If you are :ref:`running Scrapy from a script `, you can @@ -410,6 +364,38 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +.. _start-requests: + +Start requests +============== + +**Start requests** are :class:`~scrapy.Request` objects yielded from the +:meth:`~scrapy.Spider.start` method of a spider or from the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of a +:ref:`spider middleware `. + +.. seealso:: :ref:`start-request-order` + +.. _start-requests-lazy: + +Delaying start request iteration +-------------------------------- + +You can override the :meth:`~scrapy.Spider.start` method as follows to pause +its iteration whenever there are scheduled requests: + +.. code-block:: python + + async def start(self): + async for item_or_request in super().start(): + if self.crawler.engine.needs_backoff(): + await self.crawler.signals.wait_for(signals.scheduler_empty) + yield item_or_request + +This can help minimize the number of requests in the scheduler at any given +time, to minimize resource usage (memory or disk, depending on +:setting:`JOBDIR`). + .. _builtin-spiders: Generic Spiders @@ -940,10 +926,11 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - def start_requests(self): - requests = list(super(MySpider, self).start_requests()) - requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls] - return requests + async def start(self): + async for item_or_request in super().start(): + yield item_or_request + for url in self.other_urls: + yield Request(url, self.parse_other) def parse_shop(self, response): pass # ... scrape shop here ... diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 0e4a8fa6c..3e9bbe56e 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -59,6 +59,8 @@ Default Username and Password can be overridden by the settings You need the telnet program which comes installed by default in Windows, and most Linux distros. +.. _telnet-vars: + Available variables in the telnet console ========================================= @@ -77,8 +79,6 @@ convenience: +----------------+-------------------------------------------------------------------+ | ``spider`` | the active spider | +----------------+-------------------------------------------------------------------+ -| ``slot`` | the engine slot | -+----------------+-------------------------------------------------------------------+ | ``extensions`` | the Extension Manager (Crawler.extensions attribute) | +----------------+-------------------------------------------------------------------+ | ``stats`` | the Stats Collector (Crawler.stats attribute) | @@ -114,10 +114,10 @@ using the telnet console:: engine.scraper.is_idle() : False engine.spider.name : followall engine.spider_is_idle() : False - engine.slot.closing : False - len(engine.slot.inprogress) : 16 - len(engine.slot.scheduler.dqs or []) : 0 - len(engine.slot.scheduler.mqs) : 92 + engine._slot.closing : False + len(engine._slot.inprogress) : 16 + len(engine._slot.scheduler.dqs or []) : 0 + len(engine._slot.scheduler.mqs) : 92 len(engine.scraper.slot.queue) : 0 len(engine.scraper.slot.active) : 0 engine.scraper.slot.active_size : 0 diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 119dfdabb..269b27336 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,6 +34,10 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) + async def start(self): + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self): url = self.benchurl if self.latency is not None: diff --git a/pyproject.toml b/pyproject.toml index e14efdd17..187587eb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -238,7 +238,9 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", ] -filterwarnings = [] +filterwarnings = [ + "ignore::DeprecationWarning:twisted.web.static" +] [tool.ruff.lint] extend-select = [ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 16dae6ac4..96bb1ae84 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,9 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import Iterable - - from scrapy import Request + from collections.abc import AsyncIterator class Command(ScrapyCommand): @@ -61,10 +59,10 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" - return [scrapy.Request(url, dont_filter=True)] + yield scrapy.Request(url, dont_filter=True) def parse(self, response: Response) -> Any: assert isinstance(response, TextResponse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 1ce155da7..56dc1ea55 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -80,10 +80,14 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader + async def start(self): + for request in conman.from_spider(self, result): + yield request + with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] + spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: @@ -101,10 +105,10 @@ class Command(ScrapyCommand): for method in sorted(methods): print(f" * {method}") else: - start = time.time() + start_time = time.time() self.crawler_process.start() stop = time.time() result.printErrors() - result.printSummary(start, stop) + result.printSummary(start_time, stop) self.exitcode = int(not result.wasSuccessful()) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 8a8d04ff6..ef6e13de2 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -89,5 +89,11 @@ class Command(ScrapyCommand): spidercls = spider_loader.load(opts.spider) else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) + + async def start(self): + yield request + + spidercls.start = start # type: ignore[method-assign,attr-defined] + + self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c6ed20b3b..0dd9954cb 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: import argparse - from collections.abc import AsyncGenerator, Coroutine, Iterable + from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable from twisted.python.failure import Failure @@ -258,11 +258,11 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider: Spider) -> Iterable[Request]: + async def start(spider: Spider) -> AsyncIterator[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] + self.spidercls.start = start # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 3047ae396..9dabfcd9c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -24,9 +24,9 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = False default_settings = { + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } def syntax(self) -> str: @@ -85,7 +85,7 @@ class Command(ScrapyCommand): crawler._apply_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() - crawler.engine.start() + crawler.engine.start(_start_request_processing=False) self._start_crawler_thread() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 653e5e05c..7f5dd0405 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging from time import time +from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -16,15 +17,19 @@ from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals -from scrapy.core.scraper import Scraper, _HandleOutputDeferred +from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import Callable, Generator, Iterable, Iterator + from collections.abc import AsyncIterator, Callable, Generator from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -40,17 +45,15 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -class Slot: +class _Slot: def __init__( self, - start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() - self.start_requests: Iterator[Request] | None = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler @@ -78,6 +81,8 @@ class Slot: class ExecutionEngine: + _SLOT_HEARTBEAT_INTERVAL: float = 5.0 + def __init__( self, crawler: Crawler, @@ -88,20 +93,25 @@ class ExecutionEngine: self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Slot | None = None + self._slot: _Slot | None = None self.spider: Spider | None = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( - crawler.settings - ) - downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) - self.downloader: Downloader = downloader_cls(crawler) - self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = ( spider_closed_callback ) self.start_time: float | None = None + self._start: AsyncIterator[Any] | None = None + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) + try: + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( + crawler.settings + ) + self.downloader: Downloader = downloader_cls(crawler) + self.scraper: Scraper = Scraper(crawler) + except Exception: + self.close() + raise def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -114,22 +124,28 @@ class ExecutionEngine: ) return scheduler_cls - @inlineCallbacks - def start(self) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def start(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() - yield self.signals.send_catch_log_deferred(signal=signals.engine_started) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_started) + ) self.running = True self._closewait: Deferred[None] = Deferred() - yield self._closewait + if _start_request_processing: + self._start_request_processing() + await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" - @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + @deferred_f_from_coro_f + async def _finish_stopping_engine(_: Any) -> None: + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + ) self._closewait.callback(None) if not self.running: @@ -163,59 +179,85 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - def _next_request(self) -> None: - if self.slot is None: - return + async def _process_start_next(self): + """Processes the next item or request from Spider.start(). - assert self.spider is not None # typing - - if self.paused: - return - - while ( - not self._needs_backout() - and self._next_request_from_scheduler() is not None - ): - pass - - if self.slot.start_requests is not None and not self._needs_backout(): - try: - request_or_item = next(self.slot.start_requests) - except StopIteration: - self.slot.start_requests = None - except Exception: - self.slot.start_requests = None - logger.error( - "Error while obtaining start requests", - exc_info=True, - extra={"spider": self.spider}, - ) + If a request, it is scheduled. If an item, it is sent to item + pipelines. + """ + try: + item_or_request = await self._start.__anext__() + except StopAsyncIteration: + self._start = None + except Exception as exception: + self._start = None + exception_traceback = format_exc() + logger.error( + f"Error while reading start items and requests: {exception}.\n{exception_traceback}", + exc_info=True, + ) + else: + if not self.spider: + return # spider already closed + if isinstance(item_or_request, Request): + self.crawl(item_or_request) else: - if isinstance(request_or_item, Request): - self.crawl(request_or_item) - else: - self.scraper.start_itemproc(request_or_item, response=None) + self.scraper.start_itemproc(item_or_request, response=None) + self._slot.nextcall.schedule() - if self.spider_is_idle() and self.slot.close_if_idle: + @deferred_f_from_coro_f + async def _start_request_processing(self) -> None: + """Starts consuming Spider.start() output and sending scheduled + requests.""" + # Starts the processing of scheduled requests, as well as a periodic + # call to that processing method for scenarios where the scheduler + # reports having pending requests but returns none. + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + + def _start_scheduled_requests(self) -> None: + if self._slot is None or self._slot.closing is not None or self.paused: + return + + while not self.needs_backout(): + if not self._start_scheduled_request(): + break + + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() - def _needs_backout(self) -> bool: - assert self.slot is not None # typing + def needs_backout(self) -> bool: + """Returns ``True`` if no more requests can be sent at the moment, or + ``False`` otherwise. + + See :ref:`start-requests-lazy` for an example. + """ + assert self._slot is not None # typing assert self.scraper.slot is not None # typing return ( not self.running - or bool(self.slot.closing) + or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Deferred[None] | None: - assert self.slot is not None # typing + def _start_scheduled_request(self) -> bool: + assert self._slot is not None # typing assert self.spider is not None # typing - request = self.slot.scheduler.next_request() + request = self._slot.scheduler.next_request() if request is None: - return None + self.signals.send_catch_log(signals.scheduler_empty) + return False d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) @@ -228,8 +270,8 @@ class ExecutionEngine: ) def _remove_request(_: Any) -> None: - assert self.slot - self.slot.remove_request(request) + assert self._slot + self._slot.remove_request(request) d2: Deferred[None] = d.addBoth(_remove_request) d2.addErrback( @@ -239,7 +281,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - slot = self.slot + slot = self._slot d2.addBoth(lambda _: slot.nextcall.schedule()) d2.addErrback( lambda f: logger.info( @@ -248,13 +290,12 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - return d2 + return True + @inlineCallbacks def _handle_downloader_output( self, result: Request | Response | Failure, request: Request - ) -> _HandleOutputDeferred | None: - assert self.spider is not None # typing - + ) -> Generator[Deferred[Any], Any, None]: if not isinstance(result, (Request, Response, Failure)): raise TypeError( f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}" @@ -263,35 +304,35 @@ class ExecutionEngine: # downloader middleware can return requests (for example, redirects) if isinstance(result, Request): self.crawl(result) - return None + return - d = self.scraper.enqueue_scrape(result, request) - d.addErrback( - lambda f: logger.error( - "Error while enqueuing downloader output", - exc_info=failure_to_exc_info(f), + try: + yield self.scraper.enqueue_scrape(result, request) + except Exception: + assert self.spider is not None + logger.error( + "Error while enqueuing scrape", + exc_info=True, extra={"spider": self.spider}, ) - ) - return d def spider_is_idle(self) -> bool: - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False if self.downloader.active: # downloader has pending requests return False - if self.slot.start_requests is not None: # not all start requests are handled + if self._start is not None: # not all start requests are handled return False - return not self.slot.scheduler.has_pending_requests() + return not self._slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request) - self.slot.nextcall.schedule() # type: ignore[union-attr] + self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request) -> None: request_scheduled_result = self.signals.send_catch_log( @@ -303,7 +344,7 @@ class ExecutionEngine: for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return - if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=self.spider ) @@ -320,14 +361,14 @@ class ExecutionEngine: def _downloaded( self, result: Response | Request | Failure, request: Request ) -> Deferred[Response] | Response | Failure: - assert self.slot is not None # typing - self.slot.remove_request(request) + assert self._slot is not None # typing + self._slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result def _download(self, request: Request) -> Deferred[Response | Request]: - assert self.slot is not None # typing + assert self._slot is not None # typing - self.slot.add_request(request) + self._slot.add_request(request) def _on_success(result: Response | Request) -> Response | Request: if not isinstance(result, (Response, Request)): @@ -352,8 +393,8 @@ class ExecutionEngine: return result def _on_complete(_: _T) -> _T: - assert self.slot is not None - self.slot.nextcall.schedule() + assert self._slot is not None + self._slot.nextcall.schedule() return _ assert self.spider is not None @@ -362,31 +403,28 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @inlineCallbacks - def open_spider( + @deferred_f_from_coro_f + async def open_spider( self, spider: Spider, - start_requests: Iterable[Request] = (), close_if_idle: bool = True, - ) -> Generator[Deferred[Any], Any, None]: - if self.slot is not None: + ) -> None: + if self._slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) - nextcall = CallLaterOnce(self._next_request) - scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - start_requests = yield self.scraper.spidermw.process_start_requests( - start_requests, spider - ) - self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider + nextcall = CallLaterOnce(self._start_scheduled_requests) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) + self._slot = _Slot(close_if_idle, nextcall, scheduler) + self._start = await self.scraper.spidermw.process_start(spider) if hasattr(scheduler, "open") and (d := scheduler.open(spider)): - yield d - yield self.scraper.open_spider(spider) + await maybe_deferred_to_future(d) + await maybe_deferred_to_future(self.scraper.open_spider(spider)) assert self.crawler.stats self.crawler.stats.open_spider(spider) - yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - self.slot.nextcall.schedule() - self.slot.heartbeat.start(5) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) + ) def _spider_idle(self) -> None: """ @@ -415,17 +453,17 @@ class ExecutionEngine: def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") - if self.slot.closing is not None: - return self.slot.closing + if self._slot.closing is not None: + return self._slot.closing logger.info( "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - dfd = self.slot.close() + dfd = self._slot.close() def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: @@ -441,8 +479,8 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider()) dfd.addErrback(log_failure("Scraper close failure")) - if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) + if hasattr(self._slot.scheduler, "close"): + dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 4bb143dfd..57d27b7cf 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -5,13 +5,16 @@ import logging from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, cast +from warnings import warn # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: # requires queuelib >= 1.6.2 @@ -50,18 +53,17 @@ class BaseSchedulerMeta(type): class BaseScheduler(metaclass=BaseSchedulerMeta): - """ - The scheduler component is responsible for storing requests received from - the engine, and feeding them back upon request (also to the engine). + """The scheduler component is responsible for storing requests received + from the engine, and feeding them back upon request (also to the engine). The original sources of said requests are: - * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``start`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods The order in which the scheduler returns its stored requests (via the ``next_request`` method) - plays a great part in determining the order in which those requests are downloaded. + plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`. The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. """ @@ -126,55 +128,112 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): class Scheduler(BaseScheduler): + """Default scheduler. + + Requests are stored into priority queues + (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by + :attr:`~scrapy.http.Request.priority`. + + By default, a single, memory-based priority queue is used for all requests. + When using :setting:`JOBDIR`, a disk-based priority queue is also created, + and only unserializable requests are stored in the memory-based priority + queue. For a given priority value, requests in memory take precedence over + requests in disk. + + Each priority queue stores requests in separate internal queues, one per + priority value. The memory priority queue uses + :setting:`SCHEDULER_MEMORY_QUEUE` queues, while the disk priority queue + uses :setting:`SCHEDULER_DISK_QUEUE` queues. The internal queues determine + :ref:`request order ` when requests have the same priority. + :ref:`Start requests ` are stored into separate internal + queues by default, and :ref:`ordered differently `. + + Duplicate requests are filtered out with an instance of + :setting:`DUPEFILTER_CLASS`. + + .. _request-order: + + Request order + ============= + + With default settings, pending requests are stored in a LIFO_ queue + (:ref:`except for start requests `). As a result, + crawling happens in `DFO order`_, which is usually the most convenient + crawl order. However, you can enforce :ref:`BFO ` or :ref:`a custom + order ` (:ref:`except for the first few requests + `). + + .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) + .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search + + .. _start-request-order: + + Start request order + ------------------- + + :ref:`Start requests ` are sent in the order they are + yielded from :meth:`~scrapy.Spider.start`, and given the same + :attr:`~scrapy.http.Request.priority`, start requests take precedence over + other requests. + + You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests + the same as other requests when it comes to order and priority. + + + .. _bfo: + + Crawling in BFO order + --------------------- + + If you do want to crawl in `BFO order`_, you can do it by setting the + following :ref:`settings `: + + | :setting:`DEPTH_PRIORITY` = ``1`` + | :setting:`SCHEDULER_DISK_QUEUE` = ``"scrapy.squeues.PickleFifoDiskQueue"`` + | :setting:`SCHEDULER_MEMORY_QUEUE` = ``"scrapy.squeues.FifoMemoryQueue"`` + + .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search + + + .. _custom-request-order: + + Crawling in a custom order + -------------------------- + + You can manually set :attr:`~scrapy.http.Request.priority` on requests to + force a specific request order. + + + .. _concurrency-v-order: + + Concurrency affects order + ------------------------- + + While pending requests are below the configured values of + :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + concurrently. + + As a result, the first few requests of a crawl may not follow the desired + order. Lowering those settings to ``1`` enforces the desired order except + for the very first request, but it significantly slows down the crawl as a + whole. """ - Default Scrapy scheduler. This implementation also handles duplication - filtering via the :setting:`dupefilter `. - This scheduler stores requests into several priority queues (defined by the - :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues - are backed by either memory or disk based queues (respectively defined by the - :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). - - Request prioritization is almost entirely delegated to the priority queue. The only - prioritization performed by this scheduler is using the disk-based queue if present - (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based - queue if a serialization error occurs. If the disk queue is not present, the memory one - is used directly. - - :param dupefilter: An object responsible for checking and filtering duplicate requests. - The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. - :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: - any class that implements the `BaseDupeFilter` interface - - :param jobdir: The path of a directory to be used for persisting the crawl's state. - The value for the :setting:`JOBDIR` setting is used by default. - See :ref:`topics-jobs`. - :type jobdir: :class:`str` or ``None`` - - :param dqclass: A class to be used as persistent request queue. - The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. - :type dqclass: class - - :param mqclass: A class to be used as non-persistent request queue. - The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. - :type mqclass: class - - :param logunser: A boolean that indicates whether or not unserializable requests should be logged. - The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. - :type logunser: bool - - :param stats: A stats collector object to record stats about the request scheduling process. - The value for the :setting:`STATS_CLASS` setting is used by default. - :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: - any class that implements the `StatsCollector` interface - - :param pqclass: A class to be used as priority queue for requests. - The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. - :type pqclass: class - - :param crawler: The crawler object corresponding to the current crawl. - :type crawler: :class:`scrapy.crawler.Crawler` - """ + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) + return cls( + dupefilter=build_from_crawler(dupefilter_cls, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), + mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), + logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), + stats=crawler.stats, + pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), + crawler=crawler, + ) def __init__( self, @@ -187,6 +246,42 @@ class Scheduler(BaseScheduler): pqclass: type[ScrapyPriorityQueue] | None = None, crawler: Crawler | None = None, ): + """Initialize the scheduler. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ self.df: BaseDupeFilter = dupefilter self.dqdir: str | None = self._dqdir(jobdir) self.pqclass: type[ScrapyPriorityQueue] | None = pqclass @@ -195,23 +290,22 @@ class Scheduler(BaseScheduler): self.logunser: bool = logunser self.stats: StatsCollector | None = stats self.crawler: Crawler | None = crawler - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - """ - Factory method, initializes the scheduler with arguments taken from the crawl settings - """ - dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) - return cls( - dupefilter=build_from_crawler(dupefilter_cls, crawler), - jobdir=job_dir(crawler.settings), - dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), - mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), - logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), - stats=crawler.stats, - pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), - crawler=crawler, + self._sdqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "DISK" ) + self._smqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "MEMORY" + ) + + def _get_start_queue_cls( + self, crawler: Crawler | None, queue: str + ) -> type[BaseQueue] | None: + if crawler is None: + return None + cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"] + if not cls: + return None + return load_object(cls) def has_pending_requests(self) -> bool: return len(self) > 0 @@ -324,12 +418,27 @@ class Scheduler(BaseScheduler): """Create a new priority queue instance, with in-memory storage""" assert self.crawler assert self.pqclass - return build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.mqclass, - key="", - ) + try: + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + start_queue_cls=self._smqclass, + ) + except TypeError: + warn( + f"The __init__ method of {global_object_name(self.pqclass)} " + f"does not support a `start_queue_cls` keyword-only " + f"parameter.", + ScrapyDeprecationWarning, + ) + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + ) def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" @@ -337,13 +446,29 @@ class Scheduler(BaseScheduler): assert self.dqdir assert self.pqclass state = self._read_dqs_state(self.dqdir) - q = build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.dqclass, - key=self.dqdir, - startprios=state, - ) + try: + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + start_queue_cls=self._sdqclass, + ) + except TypeError: + warn( + f"The __init__ method of {global_object_name(self.pqclass)} " + f"does not support a `start_queue_cls` keyword-only " + f"parameter.", + ScrapyDeprecationWarning, + ) + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + ) if q: logger.info( "Resuming crawl (%(queuesize)d requests scheduled)", diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6f69d668e..9378f2651 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -6,10 +6,10 @@ from __future__ import annotations import logging import warnings from collections import deque -from collections.abc import AsyncIterable, Iterator -from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any, TypeVar, Union -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks, maybeDeferred from twisted.python.failure import Failure from scrapy import Spider, signals @@ -22,10 +22,12 @@ from scrapy.exceptions import ( ) from scrapy.http import Request, Response from scrapy.utils.defer import ( + _defer_sleep, aiter_errback, - defer_fail, - defer_succeed, + deferred_f_from_coro_f, + deferred_from_coro, iter_errback, + maybe_deferred_to_future, parallel, parallel_async, ) @@ -46,9 +48,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = list[tuple[bool, Iterator[Any]]] -_HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] -QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] +QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]] class Slot: @@ -66,8 +66,9 @@ class Slot: def add_response_request( self, result: Response | Failure, request: Request - ) -> _HandleOutputDeferred: - deferred: _HandleOutputDeferred = Deferred() + ) -> Deferred[None]: + # this Deferred will be awaited in enqueue_scrape() + deferred: Deferred[None] = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -76,9 +77,9 @@ class Slot: return deferred def next_response_request_deferred(self) -> QueueTuple: - response, request, deferred = self.queue.popleft() + result, request, deferred = self.queue.popleft() self.active.add(request) - return response, request, deferred + return result, request, deferred def finish_response(self, result: Response | Failure, request: Request) -> None: self.active.remove(request) @@ -143,9 +144,10 @@ class Scraper: assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) + @inlineCallbacks def enqueue_scrape( self, result: Response | Failure, request: Request, spider: Spider | None = None - ) -> _HandleOutputDeferred: + ) -> Generator[Deferred[Any], Any, None]: if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", @@ -156,103 +158,106 @@ class Scraper: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - - def finish_scraping(_: _T) -> _T: - assert self.slot is not None + self._scrape_next() + try: + yield dfd + except Exception: + logger.error( + "Scraper bug processing %(request)s", + {"request": request}, + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + finally: self.slot.finish_response(result, request) self._check_if_closing() self._scrape_next() - return _ - - dfd.addBoth(finish_scraping) - dfd.addErrback( - lambda f: logger.error( - "Scraper bug processing %(request)s", - {"request": request}, - exc_info=failure_to_exc_info(f), - extra={"spider": self.crawler.spider}, - ) - ) - self._scrape_next() - return dfd def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: - response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request).chainDeferred(deferred) + result, request, deferred = self.slot.next_response_request_deferred() + self._scrape(result, request).chainDeferred(deferred) - def _scrape( - self, result: Response | Failure, request: Request - ) -> _HandleOutputDeferred: - """ - Handle the downloaded response or failure through the spider callback/errback - """ + @deferred_f_from_coro_f + async def _scrape(self, result: Response | Failure, request: Request) -> None: + """Handle the downloaded response or failure through the spider callback/errback.""" if not isinstance(result, (Response, Failure)): raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( - result, request - ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result) - dfd2: _HandleOutputDeferred = dfd.addCallback( - self.handle_spider_output, request, cast(Response, result) - ) - return dfd2 - def _scrape2( - self, result: Response | Failure, request: Request - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: - """ - Handle the different cases of request's result been a Response or a Failure - """ + assert self.crawler.spider if isinstance(result, Response): - # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable - assert self.crawler.spider - return self.spidermw.scrape_response( # type: ignore[return-value] - self.call_spider, result, request, self.crawler.spider - ) - # else result is a Failure - dfd = self.call_spider(result, request) - dfd.addErrback(self._log_download_errors, result, request) - return dfd + try: + # call the spider middlewares and the request callback with the response + output = await maybe_deferred_to_future( + self.spidermw.scrape_response( + self.call_spider, result, request, self.crawler.spider + ) + ) + except Exception: + self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) + return + + try: + # call the request errback with the downloader error + await self.call_spider_async(result, request) + except Exception as spider_exc: + # the errback didn't silence the exception + if not result.check(IgnoreRequest): + logkws = self.logformatter.download_error( + result, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(result), + ) + if spider_exc is not result.value: + # the errback raised a different exception, handle it + self.handle_spider_error(Failure(), request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]: if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.call_spider() is deprecated.", category=ScrapyDeprecationWarning, stacklevel=2, ) + return deferred_from_coro(self.call_spider_async(result, request)) + async def call_spider_async( + self, result: Response | Failure, request: Request + ) -> Iterable[Any] | AsyncIterator[Any]: + """Call the request callback or errback with the response or failure.""" + await maybe_deferred_to_future(_defer_sleep()) assert self.crawler.spider - dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request callback = result.request.callback or self.crawler.spider._parse warn_on_generator_with_return_value(self.crawler.spider, callback) - dfd = defer_succeed(result) - dfd.addCallbacks( - callback=callback, callbackKeywords=result.request.cb_kwargs - ) + output = callback(result, **result.request.cb_kwargs) else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] - dfd = defer_fail(result) - if request.errback: - warn_on_generator_with_return_value( - self.crawler.spider, request.errback - ) - dfd.addErrback(request.errback) - dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( - iterate_spider_output + if not request.errback: + result.raiseException() + warn_on_generator_with_return_value(self.crawler.spider, request.errback) + output = request.errback(result) + if isinstance(output, Failure): + output.raiseException() + # else the errback returned actual output (like a callback), + # which needs to be passed to iterate_spider_output() + return await maybe_deferred_to_future( + maybeDeferred(iterate_spider_output, output) ) - return dfd2 def handle_spider_error( self, @@ -261,6 +266,7 @@ class Scraper: response: Response | Failure, spider: Spider | None = None, ) -> None: + """Handle an exception raised by a spider callback or errback.""" if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", @@ -301,57 +307,68 @@ class Scraper: def handle_spider_output( self, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], request: Request, response: Response, spider: Spider | None = None, - ) -> _HandleOutputDeferred: + ) -> Deferred[None]: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.", category=ScrapyDeprecationWarning, stacklevel=2, ) + return deferred_from_coro( + self.handle_spider_output_async(result, request, response) + ) - if not result: - return defer_succeed(None) - it: Iterable[_T] | AsyncIterable[_T] - dfd: Deferred[_ParallelResult] - if isinstance(result, AsyncIterable): - it = aiter_errback(result, self.handle_spider_error, request, response) - dfd = parallel_async( + async def handle_spider_output_async( + self, + result: Iterable[_T] | AsyncIterator[_T], + request: Request, + response: Response, + ) -> None: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" + if isinstance(result, AsyncIterator): + ait = aiter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel_async( + ait, + self.concurrent_items, + self._process_spidermw_output, + response, + ) + ) + return + it = iter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel( it, self.concurrent_items, self._process_spidermw_output, response, ) - else: - it = iter_errback(result, self.handle_spider_error, request, response) - dfd = parallel( - it, - self.concurrent_items, - self._process_spidermw_output, - response, - ) - # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] - return dfd # type: ignore[return-value] + ) - def _process_spidermw_output( - self, output: Any, response: Response - ) -> Deferred[Any] | None: + @deferred_f_from_coro_f + async def _process_spidermw_output(self, output: Any, response: Response) -> None: """Process each Request/Item (given in the output parameter) returned - from the given spider + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. """ if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif output is None: - pass - else: - return self.start_itemproc(output, response=response) - return None + return + if output is not None: + await maybe_deferred_to_future( + self.start_itemproc(output, response=response) + ) - def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: + @deferred_f_from_coro_f + async def start_itemproc(self, item: Any, *, response: Response | None) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -360,90 +377,56 @@ class Scraper: assert self.slot is not None # typing assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(item, self.crawler.spider) - dfd.addBoth(self._itemproc_finished, item, response) - return dfd - - def _log_download_errors( - self, - spider_failure: Failure, - download_failure: Failure, - request: Request, - ) -> Failure | None: - """Log and silence errors that come from the engine (typically download - errors that got propagated thru here). - - spider_failure: the value passed into the errback of self.call_spider() - (likely raised in the request errback) - - download_failure: the value passed into _scrape2() from - ExecutionEngine._handle_downloader_output() as "result" - (likely raised in the download handler or a downloader middleware) - """ - if not download_failure.check(IgnoreRequest): - assert self.crawler.spider - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider + try: + output = await maybe_deferred_to_future( + self.itemproc.process_item(item, self.crawler.spider) ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(download_failure), - ) - if spider_failure is not download_failure: - # a request errback raised a different exception, it needs to be handled later - return spider_failure - return None - - def _itemproc_finished( - self, output: Any, item: Any, response: Response | None - ) -> Deferred[Any]: - """ItemProcessor finished for the given ``item`` and returned ``output``""" - assert self.slot is not None # typing - assert self.crawler.spider - self.slot.itemproc_size -= 1 - if isinstance(output, Failure): - ex = output.value - if isinstance(ex, DropItem): - logkws = self.logformatter.dropped( - item, ex, response, self.crawler.spider + except DropItem as ex: + logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - if logkws is not None: - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - ) - return self.signals.send_catch_log_deferred( + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=self.crawler.spider, - exception=output.value, + exception=ex, ) - assert ex + ) + except Exception as ex: logkws = self.logformatter.item_error( item, ex, response, self.crawler.spider ) logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(output), + exc_info=True, ) - return self.signals.send_catch_log_deferred( - signal=signals.item_error, - item=item, - response=response, - spider=self.crawler.spider, - failure=output, + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( + signal=signals.item_error, + item=item, + response=response, + spider=self.crawler.spider, + failure=Failure(), + ) ) - logkws = self.logformatter.scraped(output, response, self.crawler.spider) - if logkws is not None: - logger.log( - *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + else: + logkws = self.logformatter.scraped(output, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, + ) ) - return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, - item=output, - response=response, - spider=self.crawler.spider, - ) + finally: + self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b8b0aec44..4a0cd9464 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,16 +7,17 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, Callable, Iterable +from collections.abc import AsyncIterator, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from warnings import warn from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen @@ -40,12 +41,13 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request], + Deferred[Union[Iterable[_T], AsyncIterator[_T]]], ] def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterable)) + return isinstance(o, (Iterable, AsyncIterator)) class SpiderMiddlewareManager(MiddlewareManager): @@ -55,12 +57,75 @@ class SpiderMiddlewareManager(MiddlewareManager): def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) + def __init__(self, *middlewares: Any) -> None: + self._check_deprecated_process_start_requests_use(middlewares) + super().__init__(*middlewares) + + def _check_deprecated_process_start_requests_use( + self, middlewares: tuple[Any] + ) -> None: + deprecated_middlewares = [ + middleware + for middleware in middlewares + if hasattr(middleware, "process_start_requests") + and not hasattr(middleware, "process_start") + ] + modern_middlewares = [ + middleware + for middleware in middlewares + if not hasattr(middleware, "process_start_requests") + and hasattr(middleware, "process_start") + ] + if deprecated_middlewares and modern_middlewares: + raise ValueError( + "You are trying to combine spider middlewares that only " + "define the deprecated process_start_requests() method () " + "with spider middlewares that only define the " + "process_start() method (). This is not possible. You must " + "either disable or make universal 1 of those 2 sets of " + "spider middlewares. Making a spider middleware universal " + "means having it define both methods. See the release notes " + "of Scrapy VERSION for details: " + "https://docs.scrapy.org/en/VERSION/news.html" + ) + + self._use_start_requests = bool(deprecated_middlewares) + if self._use_start_requests: + deprecated_middleware_list = ", ".join( + global_object_name(middleware.__class__) + for middleware in deprecated_middlewares + ) + warn( + f"The following enabled spider middlewares, directly or " + f"through their parent classes, define the deprecated " + f"process_start_requests() method: " + f"{deprecated_middleware_list}. process_start_requests() has " + f"been deprecated in favor of a new method, process_start(), " + f"to support asynchronous code execution. " + f"process_start_requests() will stop being called in a future " + f"version of Scrapy. If you use Scrapy VERSION or higher " + f"only, replace process_start_requests() with " + f"process_start(); note that process_start() is a coroutine " + f"(async def). If you need to maintain compatibility with " + f"lower Scrapy versions, when defining " + f"process_start_requests() in a spider middleware class, " + f"define process_start() as well. See the release notes of " + f"Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + def _add_middleware(self, mw: Any) -> None: super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft(mw.process_start_requests) + if self._use_start_requests: + if hasattr(mw, "process_start_requests"): + self.methods["process_start_requests"].appendleft( + mw.process_start_requests + ) + elif hasattr(mw, "process_start"): + self.methods["process_start"].appendleft(mw.process_start) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) process_spider_exception = getattr(mw, "process_spider_exception", None) @@ -72,7 +137,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Deferred[Iterable[_T] | AsyncIterator[_T]]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -93,10 +158,10 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Iterable[_T] | AsyncIterable[_T], + iterable: Iterable[_T] | AsyncIterator[_T], exception_processor_index: int, recover_to: MutableChain[_T] | MutableAsyncChain[_T], - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Iterable[_T] | AsyncIterator[_T]: def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable @@ -112,7 +177,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: + async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]: try: async for r in iterable: yield r @@ -128,7 +193,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) - if isinstance(iterable, AsyncIterable): + if isinstance(iterable, AsyncIterator): return process_async(iterable) return process_sync(iterable) @@ -187,13 +252,13 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], start_index: int = 0, ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. @@ -220,7 +285,7 @@ class SpiderMiddlewareManager(MiddlewareManager): need_downgrade = True try: if need_upgrade: - # Iterable -> AsyncIterable + # Iterable -> AsyncIterator result = as_async_generator(result) elif need_downgrade: logger.warning( @@ -230,10 +295,10 @@ class SpiderMiddlewareManager(MiddlewareManager): f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) - assert isinstance(result, AsyncIterable) - # AsyncIterable -> Iterable + assert isinstance(result, AsyncIterator) + # AsyncIterator -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = yield deferred_from_coro( collect_asyncgen(recovered) ) @@ -266,7 +331,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"{type(result)}" ) raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) if last_result_is_async: return MutableAsyncChain(result, recovered) @@ -276,23 +341,23 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: recovered: MutableChain[_T] | MutableAsyncChain[_T] - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( cast( - "Deferred[Iterable[_T] | AsyncIterable[_T]]", + "Deferred[Iterable[_T] | AsyncIterator[_T]]", self._process_spider_output(response, spider, result), ) ) - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = await collect_asyncgen(recovered) recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) @@ -305,7 +370,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: async def process_callback_output( - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) @@ -314,7 +379,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred[Iterable[_T] | AsyncIterable[_T]] = mustbe_deferred( + dfd: Deferred[Iterable[_T] | AsyncIterator[_T]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( @@ -323,10 +388,90 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd2.addErrback(process_spider_exception) return dfd2 - def process_start_requests( - self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred[Iterable[Request]]: - return self._process_chain("process_start_requests", start_requests, spider) + async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: + self._check_deprecated_start_requests_use(spider) + if self._use_start_requests: + sync_start = iter(spider.start_requests()) + sync_start = await maybe_deferred_to_future( + self._process_chain("process_start_requests", sync_start, spider) + ) + start: AsyncIterator[Any] = as_async_generator(sync_start) + else: + start = spider.start() + start = await maybe_deferred_to_future( + self._process_chain("process_start", start) + ) + return start + + def _check_deprecated_start_requests_use(self, spider: Spider): + start_requests_cls = None + start_cls = None + spidercls = spider.__class__ + mro = spidercls.__mro__ + + for cls in mro: + cls_dict = cls.__dict__ + if start_requests_cls is None and "start_requests" in cls_dict: + start_requests_cls = cls + if start_cls is None and "start" in cls_dict: + start_cls = cls + if start_requests_cls is not None and start_cls is not None: + break + + # Spider defines both, start_requests and start. + assert start_requests_cls is not None + assert start_cls is not None + + if ( + start_requests_cls is not Spider + and start_cls is not start_requests_cls + and mro.index(start_requests_cls) < mro.index(start_cls) + ): + src = global_object_name(start_requests_cls) + if start_requests_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + warn( + f"{src} defines the deprecated start_requests() method. " + f"start_requests() has been deprecated in favor of a new " + f"method, start(), to support asynchronous code " + f"execution. start_requests() will stop being called in a " + f"future version of Scrapy. If you use Scrapy VERSION or " + f"higher only, replace start_requests() with start(); " + f"note that start() is a coroutine (async def). If you " + f"need to maintain compatibility with lower Scrapy versions, " + f"when overriding start_requests() in a spider class, " + f"override start() as well; you can use super() to " + f"reuse the inherited start() implementation without " + f"copy-pasting. See the release notes of Scrapy VERSION for " + f"details: https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + + if ( + self._use_start_requests + and start_cls is not Spider + and start_requests_cls is not start_cls + and mro.index(start_cls) < mro.index(start_requests_cls) + ): + src = global_object_name(start_cls) + if start_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + raise ValueError( + f"{src} does not define the deprecated start_requests() " + f"method. However, one or more of your enabled spider " + f"middlewares (reported in an earlier deprecation warning) " + f"define the process_start_requests() method, and not the " + f"process_start() method, making them only compatible with " + f"(deprecated) spiders that define the start_requests() " + f"method. To solve this issue, disable the offending spider " + f"middlewares, upgrade them as described in that earlier " + f"deprecation warning, or make your spider compatible with " + f"deprecated spider middlewares (and earlier Scrapy versions) " + f"by defining a sync start_requests() method that works " + f"similarly to its existing start() method. See the " + f"release notes of Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html" + ) # This method is only needed until _async compatibility methods are removed. @staticmethod diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1ec1e31dc..749096db5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -136,6 +136,9 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + # Cannot use @deferred_f_from_coro_f because that relies on the reactor + # being installed already, which is done within _apply_settings(), inside + # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: @@ -151,9 +154,8 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - start_requests = iter(self.spider.start_requests()) - yield self.engine.open_spider(self.spider, start_requests) - yield maybeDeferred(self.engine.start) + yield self.engine.open_spider(self.spider) + yield self.engine.start() except Exception: self.crawling = False if self.engine is not None: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ac832e025..bacee8f0a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -104,7 +104,6 @@ class TelnetConsole(protocol.ServerFactory): telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, - "slot": self.crawler.engine.slot, "crawler": self.crawler, "extensions": self.crawler.extensions, "stats": self.crawler.stats, diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6d3b7a926..2b8d0ab84 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -130,6 +130,16 @@ class Request(object_ref): self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") + + #: Default: ``0`` + #: + #: Value that the :ref:`scheduler ` may use for + #: request prioritization. + #: + #: Built-in schedulers prioritize requests with a higher priority + #: value. + #: + #: Negative values are allowed. self.priority: int = priority if not (callable(callback) or callback is None): @@ -191,7 +201,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.start_requests`. + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index f10e91beb..4f08918ae 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -98,7 +98,7 @@ class LogFormatter: """Logs a message when an item is scraped by a spider.""" src: Any if response is None: - src = f"{global_object_name(spider.__class__)}.start_requests" + src = f"{global_object_name(spider.__class__)}.start" elif isinstance(response, Failure): src = response.getErrorMessage() else: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 324a9b955..e6c6b8bf1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -72,7 +72,6 @@ class ScrapyPriorityQueue: startprios is a sequence of priorities to start with. If the queue was previously closed leaving some priority buckets non-empty, those priorities should be passed in startprios. - """ @classmethod @@ -82,8 +81,16 @@ class ScrapyPriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: - return cls(crawler, downstream_queue_cls, key, startprios) + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) def __init__( self, @@ -91,11 +98,15 @@ class ScrapyPriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, ): self.crawler: Crawler = crawler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.queues: dict[int, QueueProtocol] = {} + self._start_queues: dict[int, QueueProtocol] = {} self.curprio: int | None = None self.init_prios(startprios) @@ -104,7 +115,13 @@ class ScrapyPriorityQueue: return for priority in startprios: - self.queues[priority] = self.qfactory(priority) + q = self.qfactory(priority) + if q: + self.queues[priority] = q + if self._start_queue_cls: + q = self._sqfactory(priority) + if q: + self._start_queues[priority] = q self.curprio = min(startprios) @@ -115,29 +132,66 @@ class ScrapyPriorityQueue: self.key + "/" + str(key), ) + def _sqfactory(self, key: int) -> QueueProtocol: + assert self._start_queue_cls is not None + return build_from_crawler( + self._start_queue_cls, + self.crawler, + f"{self.key}/{key}s", + ) + def priority(self, request: Request) -> int: return -request.priority def push(self, request: Request) -> None: priority = self.priority(request) - if priority not in self.queues: - self.queues[priority] = self.qfactory(priority) - q = self.queues[priority] + is_start_request = request.meta.get("is_start_request", False) + if is_start_request and self._start_queue_cls: + if priority not in self._start_queues: + self._start_queues[priority] = self._sqfactory(priority) + q = self._start_queues[priority] + else: + if priority not in self.queues: + self.queues[priority] = self.qfactory(priority) + q = self.queues[priority] q.push(request) # this may fail (eg. serialization error) if self.curprio is None or priority < self.curprio: self.curprio = priority def pop(self) -> Request | None: - if self.curprio is None: - return None - q = self.queues[self.curprio] - m = q.pop() - if not q: - del self.queues[self.curprio] - q.close() - prios = [p for p, q in self.queues.items() if q] - self.curprio = min(prios) if prios else None - return m + while self.curprio is not None: + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + pass + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + return m + try: + q = self.queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self.queues[self.curprio] + q.close() + self._update_curprio() + return m + return None + + def _update_curprio(self) -> None: + prios = { + p + for queues in (self.queues, self._start_queues) + for p, q in queues.items() + if q + } + self.curprio = min(prios) if prios else None def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, @@ -148,19 +202,31 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None - queue = self.queues[self.curprio] + try: + queue = self._start_queues[self.curprio] + except KeyError: + queue = self.queues[self.curprio] # Protocols can't declare optional members return cast(Request, queue.peek()) # type: ignore[attr-defined] def close(self) -> list[int]: - active: list[int] = [] - for p, q in self.queues.items(): - active.append(p) - q.close() - return active + active: set[int] = set() + for queues in (self.queues, self._start_queues): + for p, q in queues.items(): + active.add(p) + q.close() + return list(active) def __len__(self) -> int: - return sum(len(x) for x in self.queues.values()) if self.queues else 0 + return ( + sum( + len(x) + for queues in (self.queues, self._start_queues) + for x in queues.values() + ) + if self.queues or self._start_queues + else 0 + ) class DownloaderInterface: @@ -194,8 +260,16 @@ class DownloaderAwarePriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: - return cls(crawler, downstream_queue_cls, key, startprios) + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) def __init__( self, @@ -203,6 +277,8 @@ class DownloaderAwarePriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, slot_startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -222,6 +298,7 @@ class DownloaderAwarePriorityQueue: self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.crawler: Crawler = crawler @@ -237,6 +314,7 @@ class DownloaderAwarePriorityQueue: self.downstream_queue_cls, self.key + "/" + _path_safe(slot), startprios, + start_queue_cls=self._start_queue_cls, ) def pop(self) -> Request | None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 680fded7a..01443fa17 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,6 +305,8 @@ SCHEDULER = "scrapy.core.scheduler.Scheduler" SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" +SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" +SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 @@ -315,6 +317,7 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side + "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, diff --git a/scrapy/shell.py b/scrapy/shell.py index 5e5e57a9a..bb39eccc3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,6 +24,7 @@ from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser @@ -102,25 +103,33 @@ class Shell: # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] set_asyncio_event_loop(event_loop_path) - spider = self._open_spider(request, spider) + + def crawl_request(_): + assert self.crawler.engine is not None + self.crawler.engine.crawl(request) + + d2 = self._open_spider(request, spider) + d2.addCallback(crawl_request) + d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - assert self.crawler.engine - self.crawler.engine.crawl(request) return d - def _open_spider(self, request: Request, spider: Spider | None) -> Spider: + @deferred_f_from_coro_f + async def _open_spider(self, request: Request, spider: Spider | None) -> None: if self.spider: - return self.spider + return if spider is None: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider assert self.crawler.engine - self.crawler.engine.open_spider(spider, close_if_idle=False) + await maybe_deferred_to_future( + self.crawler.engine.open_spider(spider, close_if_idle=False) + ) + self.crawler.engine._start_request_processing() self.spider = spider - return spider def fetch( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index e106418d6..f8c50b5e3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,13 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from pydispatch import dispatcher +from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal - -if TYPE_CHECKING: - from twisted.internet.defer import Deferred +from scrapy.utils.defer import maybe_deferred_to_future class SignalManager: @@ -75,3 +74,17 @@ class SignalManager: """ kwargs.setdefault("sender", self.sender) _signal.disconnect_all(signal, **kwargs) + + async def wait_for(self, signal): + """Await the next *signal*. + + See :ref:`start-requests-lazy` for an example. + """ + d = Deferred() + + def handle(): + self.disconnect(handle, signal) + d.callback(None) + + self.connect(handle, signal) + await maybe_deferred_to_future(d) diff --git a/scrapy/signals.py b/scrapy/signals.py index 8ef0f34f0..bdeec1ba0 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -7,6 +7,7 @@ signals here without documenting them there. engine_started = object() engine_stopped = object() +scheduler_empty = object() spider_opened = object() spider_idle = object() spider_closed = object() diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 650192095..cfb50c599 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -17,9 +17,9 @@ if TYPE_CHECKING: class BaseSpiderMiddleware: """Optional base class for spider middlewares. - This class provides helper methods for asynchronous ``process_spider_output`` - methods. Middlewares that don't have a ``process_spider_output`` method don't need - to use it. + This class provides helper methods for asynchronous + ``process_spider_output()`` and ``process_start()`` methods. Middlewares + that don't have either of these methods don't need to use this class. You can override the :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` @@ -38,59 +38,70 @@ class BaseSpiderMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) + def process_start_requests( + self, start: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: + async for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider ) -> Iterable[Any]: for o in result: - if isinstance(o, Request): - o = self.get_processed_request(o, response) - else: - o = self.get_processed_item(o, response) - if o is not None: + if (o := self._get_processed(o, response)) is not None: yield o async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: async for o in result: - if isinstance(o, Request): - o = self.get_processed_request(o, response) - else: - o = self.get_processed_item(o, response) - if o is not None: + if (o := self._get_processed(o, response)) is not None: yield o + def _get_processed(self, o: Any, response: Response | None) -> Any: + if isinstance(o, Request): + return self.get_processed_request(o, response) + return self.get_processed_item(o, response) + def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: """Return a processed request from the spider output. - This method is called with a single request from the spider output. - It should return the same or a different request, or ``None`` to - ignore it. + This method is called with a single request from the start seeds or the + spider output. It should return the same or a different request, or + ``None`` to ignore it. :param request: the input request :type request: :class:`~scrapy.Request` object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds :return: the processed request or ``None`` """ return request - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: """Return a processed item from the spider output. - This method is called with a single item from the spider output. - It should return the same or a different item, or ``None`` to - ignore it. + This method is called with a single item from the start seeds or the + spider output. It should return the same or a different item, or + ``None`` to ignore it. :param item: the input item :type item: item object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 65905f483..6b115ebe6 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -59,8 +59,8 @@ class DepthMiddleware(BaseSpiderMiddleware): yield from super().process_spider_output(response, result, spider) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: self._init_depth(response, spider) async for o in super().process_spider_output_async(response, result, spider): yield o @@ -73,8 +73,11 @@ class DepthMiddleware(BaseSpiderMiddleware): self.stats.inc_value("request_depth_count/0", spider=spider) def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # start requests + return request depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 0918c9fac..2463275d5 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -49,8 +49,11 @@ class OffsiteMiddleware(BaseSpiderMiddleware): return o def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # skip start requests for backward compatibility + return request assert self.crawler.spider if ( request.dont_filter diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index b2ba8ba8c..f5d406c13 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -370,8 +370,11 @@ class RefererMiddleware(BaseSpiderMiddleware): return cls() if cls else self.default_policy() def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # start requests + return request referrer = self.policy(response, request).referrer(response.url, request.url) if referrer is not None: request.headers.setdefault("Referer", referrer) diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.py new file mode 100644 index 000000000..5d76b60d2 --- /dev/null +++ b/scrapy/spidermiddlewares/start.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .base import BaseSpiderMiddleware + +if TYPE_CHECKING: + from scrapy.http import Request + from scrapy.http.response import Response + + +class StartSpiderMiddleware(BaseSpiderMiddleware): + """Set :reqmeta:`is_start_request`. + + .. reqmeta:: is_start_request + + is_start_request + ---------------- + + :attr:`~scrapy.Request.meta` key that is set to ``True`` in :ref:`start + requests `, allowing you to tell start requests apart from + other requests, e.g. in :ref:`downloader middlewares + `. + """ + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + request.meta.setdefault("is_start_request", True) + return request diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 177c19e1b..5590165a5 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -39,7 +39,7 @@ class UrlLengthMiddleware(BaseSpiderMiddleware): return o def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if len(request.url) <= self.maxlength: return request diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e255e91cc..0a1d85ae6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,15 +7,17 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import AsyncIterator, Iterable from twisted.internet.defer import Deferred @@ -29,13 +31,19 @@ if TYPE_CHECKING: class Spider(object_ref): - """Base class for scrapy spiders. All spiders must inherit from this - class. + """Base class that any spider must subclass. + + It provides a default :meth:`start` implementation that sends + requests based on the :attr:`start_urls` class attribute and calls the + :meth:`parse` method for each response. """ name: str custom_settings: dict[_SettingsKeyT, Any] | None = None + #: Start URLs. See :meth:`start`. + start_urls: list[str] + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name @@ -72,7 +80,70 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: + """Yield the initial :class:`~scrapy.Request` objects to send. + + .. versionadded:: VERSION + + For example: + + .. code-block:: python + + from scrapy import Request, Spider + + + class MySpider(Spider): + name = "myspider" + + async def start(self): + yield Request("https://toscrape.com/") + + The default implementation reads URLs from :attr:`start_urls` and + yields a request for each with :attr:`~scrapy.Request.dont_filter` + enabled. It is functionally equivalent to: + + .. code-block:: python + + async def start(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + + You can also yield :ref:`items `. For example: + + .. code-block:: python + + async def start(self): + yield {"foo": "bar"} + + To write spiders that work on Scrapy versions lower than VERSION, + define also a synchronous ``start_requests()`` method that returns an + iterable. For example: + + .. code-block:: python + + def start_requests(self): + yield Request("https://toscrape.com/") + + .. seealso:: :ref:`start-requests` + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request + + def start_requests(self) -> Iterable[Any]: + warnings.warn( + ( + "The Spider.start_requests() method is deprecated, use " + "Spider.start() instead. If you are calling " + "super().start_requests() from a Spider.start() override, " + "iterate super().start() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 087049425..171d8479c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from collections.abc import AsyncIterable, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast from twisted.python.failure import Failure @@ -156,10 +156,10 @@ class CrawlSpider(Spider): callback: CallbackT | None, cb_kwargs: dict[str, Any], follow: bool = True, - ) -> AsyncIterable[Any]: + ) -> AsyncIterator[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, AsyncIterable): + if isinstance(cb_res, AsyncIterator): cb_res = await collect_asyncgen(cb_res) elif isinstance(cb_res, Awaitable): cb_res = await cb_res diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a7dba989e..e5548b9fa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from collections.abc import Iterable +from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -29,6 +29,14 @@ class InitSpider(Spider): stacklevel=2, ) + async def start(self) -> AsyncIterator[Any]: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() return cast(Iterable[Request], iterate_spider_output(self.init_request())) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 39033ac3c..2813a32a0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,6 +53,10 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + async def start(self) -> AsyncIterator[Any]: + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index dcb2d63de..3f0239832 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,14 +43,11 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r + async def process_start(self, start): + # Called with an async iterator over the spider start() method or the + # maching method of an earlier spider middleware. + async for item_or_request in start: + yield item_or_request def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 237bd8331..6d96a41f5 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,20 +1,20 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from collections.abc import AsyncGenerator, AsyncIterator, Iterable from typing import TypeVar _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: +async def collect_asyncgen(result: AsyncIterator[_T]) -> list[_T]: return [x async for x in result] async def as_async_generator( - it: Iterable[_T] | AsyncIterable[_T], + it: Iterable[_T] | AsyncIterator[_T], ) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" - if isinstance(it, AsyncIterable): + if isinstance(it, AsyncIterator): async for r in it: yield r else: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 42ad28d8d..6e1687f3e 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -14,7 +14,11 @@ from types import CoroutineType from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload from twisted.internet import defer -from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.defer import ( + Deferred, + DeferredList, + ensureDeferred, +) from twisted.internet.task import Cooperator from twisted.python import failure @@ -22,7 +26,7 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: - from collections.abc import AsyncIterable, AsyncIterator, Callable + from collections.abc import AsyncIterator, Callable from twisted.python.failure import Failure @@ -36,6 +40,9 @@ _T = TypeVar("_T") _T2 = TypeVar("_T2") +_DEFER_DELAY = 0.1 + + def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -46,7 +53,7 @@ def defer_fail(_failure: Failure) -> Deferred[Any]: from twisted.internet import reactor d: Deferred[Any] = Deferred() - reactor.callLater(0.1, d.errback, _failure) + reactor.callLater(_DEFER_DELAY, d.errback, _failure) return d @@ -60,7 +67,16 @@ def defer_succeed(result: _T) -> Deferred[_T]: from twisted.internet import reactor d: Deferred[_T] = Deferred() - reactor.callLater(0.1, d.callback, result) + reactor.callLater(_DEFER_DELAY, d.callback, result) + return d + + +def _defer_sleep() -> Deferred[None]: + """Like ``defer_succeed`` and ``defer_fail`` but doesn't call any real callbacks.""" + from twisted.internet import reactor + + d: Deferred[None] = Deferred() + reactor.callLater(_DEFER_DELAY, d.callback, None) return d @@ -177,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, @@ -234,7 +250,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( - async_iterable: AsyncIterable[_T], + async_iterable: AsyncIterator[_T], count: int, callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *args: _P.args, @@ -332,13 +348,13 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], errback: Callable[Concatenate[Failure, _P], Any], *a: _P.args, **kw: _P.kwargs, -) -> AsyncIterable[_T]: +) -> AsyncIterator[_T]: """Wraps an async iterable calling an errback if an error is caught while - iterating it. Similar to scrapy.utils.defer.iter_errback() + iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`. """ it = aiterable.__aiter__() while True: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 52f29e22c..1e0c53212 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -18,10 +18,10 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: "engine.scraper.is_idle()", "engine.spider.name", "engine.spider_is_idle()", - "engine.slot.closing", - "len(engine.slot.inprogress)", - "len(engine.slot.scheduler.dqs or [])", - "len(engine.slot.scheduler.mqs)", + "engine._slot.closing", + "len(engine._slot.inprogress)", + "len(engine._slot.scheduler.dqs or [])", + "len(engine._slot.scheduler.mqs)", "len(engine.scraper.slot.queue)", "len(engine.scraper.slot.active)", "engine.scraper.slot.active_size", diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 2e6869779..c859fbc2a 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import re import sys import warnings import weakref -from collections.abc import AsyncIterable, Iterable, Mapping +from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, TypeVar, overload @@ -19,11 +19,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator + from collections.abc import Callable, Iterator from re import Pattern # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Self requires Python 3.11 + from typing_extensions import Concatenate, ParamSpec, Self _P = ParamSpec("_P") @@ -369,25 +370,25 @@ class MutableChain(Iterable[_T]): async def _async_chain( - *iterables: Iterable[_T] | AsyncIterable[_T], + *iterables: Iterable[_T] | AsyncIterator[_T], ) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable[_T]): +class MutableAsyncChain(AsyncIterator[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Iterable[_T] | AsyncIterable[_T]): + def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]): self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Iterable[_T] | AsyncIterable[_T]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterator[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator[_T]: + def __aiter__(self) -> Self: return self async def __anext__(self) -> _T: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 099c81f0e..9c2754394 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error +from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object @@ -54,6 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: DelayedCall | None = None + self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor @@ -66,8 +68,23 @@ class CallLaterOnce(Generic[_T]): self._call.cancel() def __call__(self) -> _T: + from twisted.internet import reactor + self._call = None - return self._func(*self._a, **self._kw) + result = self._func(*self._a, **self._kw) + + for d in self._deferreds: + reactor.callLater(0, d.callback, None) + self._deferreds = [] + + return result + + async def wait(self): + from scrapy.utils.defer import maybe_deferred_to_future + + d = Deferred() + self._deferreds.append(d) + await maybe_deferred_to_future(d) def set_asyncio_event_loop_policy() -> None: @@ -114,8 +131,10 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) + event_loop = _get_asyncio_event_loop() + if not isinstance(event_loop, event_loop_class): + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: try: with catch_warnings(): diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e6d601fe1..29b1f860e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -619,7 +619,7 @@ Resolved: ``manager.scraper.process_request()`` instead of ``manager.engine.crawl()`` - should we support adding additional start requests from a spider middleware? - - Yes - there is a spider middleware method (``start_requests``) for that + - Yes - there is a spider middleware method (``start_requests()``) for that - should ``process_response()`` receive a ``request`` argument with the ``request`` that originated it?. ``response.request`` is the latest request, not the original one (think of redirections), but it does carry the ``meta`` diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index a46a8806b..c8a3d0a5b 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,9 +13,10 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 5e72aa6d4..bd78a0de7 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6f82cf589..6bb6fb3c6 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -12,8 +12,9 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index a8bf1bc3c..f3dab12fe 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -38,8 +38,9 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index e9d6d8875..d8c467f40 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -15,8 +15,9 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index c72a0a17c..e7d3ca9cc 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -16,8 +16,9 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 7b0497bde..53d427061 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - def start_requests(self): + async def start(self): yield scrapy.Request(self.url) def parse(self, response): diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 9f7eaf2ae..0058896b5 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index e2933338b..8f59c035c 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index eee808c32..9901dd634 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index b61e52625..53941568a 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -10,8 +10,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 38ca4c4f1..5739d77ae 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,8 +17,9 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index b397608ec..c488f7526 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,8 +9,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 2d8769501..9e4ad70d9 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index de76e13e8..6c0102241 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -10,8 +10,9 @@ class NoRequestsSpider(Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } - def start_requests(self): - return [] + async def start(self): + return + yield configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b1b297777..5e2184afb 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -32,7 +32,7 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - def start_requests(self): + async def start(self): yield Request(self.url) def parse(self, response): diff --git a/tests/__init__.py b/tests/__init__.py index cd52ade58..ccfabb0da 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,6 +8,9 @@ import os import socket from pathlib import Path +from twisted import version as TWISTED_VERSION +from twisted.python.versions import Version + # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server # (e.g. because urllib does not fully understand the proxy spec) @@ -30,3 +33,6 @@ except socket.gaierror: def get_testdata(*paths: str) -> bytes: """Return test data""" return Path(tests_datadir, *paths).read_bytes() + + +TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) diff --git a/tests/spiders.py b/tests/spiders.py index da923de6e..c47f2bd2b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -68,7 +68,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - def start_requests(self): + async def start(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -105,7 +105,7 @@ class LogSpider(MetaSpider): class SlowSpider(DelaySpider): name = "slow" - def start_requests(self): + async def start(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -255,7 +255,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - def start_requests(self): + async def start(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenStartRequestsSpider(FollowAllSpider): +class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -327,7 +327,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - def start_requests(self): + async def start(self): if self.fail_before_yield: 1 / 0 @@ -338,22 +338,20 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, ( - "All start requests consumed before any download happened" - ) + assert self.seedsseen, "All seeds consumed before any download happened" def parse(self, response): self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) -class StartRequestsItemSpider(FollowAllSpider): - def start_requests(self): +class StartItemSpider(FollowAllSpider): + async def start(self): yield {"name": "test item"} -class StartRequestsGoodAndBadOutput(FollowAllSpider): - def start_requests(self): +class StartGoodAndBadOutput(FollowAllSpider): + async def start(self): yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -365,7 +363,7 @@ class SingleRequestSpider(MetaSpider): callback_func = None errback_func = None - def start_requests(self): + async def start(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -386,13 +384,13 @@ class SingleRequestSpider(MetaSpider): return None -class DuplicateStartRequestsSpider(MockServerSpider): +class DuplicateStartSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 dupe_factor = 3 - def start_requests(self): + async def start(self): for i in range(self.distinct_urls): for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") @@ -417,7 +415,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" Page title<title></head> @@ -471,7 +469,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" <html> <head><title>Page title<title></head> @@ -516,7 +514,7 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - def start_requests(self): + async def start(self): body = b"a" * self.full_response_length url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=body, errback=self.errback) @@ -545,7 +543,7 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status"), errback=self.errback) def parse(self, response): diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5228f6abd..5006e3689 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,17 +2,26 @@ import sys from pathlib import Path from subprocess import PIPE, Popen +from .. import TWISTED_KEEPS_TRACEBACKS + class TestCmdlineCrawlPipeline: def _execute(self, spname): args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) - proc.communicate() - return proc.returncode + _, stderr = proc.communicate() + return proc.returncode, stderr def test_open_spider_normally_in_pipeline(self): - assert self._execute("normal") == 0 + returncode, stderr = self._execute("normal") + assert returncode == 0 def test_exception_at_open_spider_in_pipeline(self): - assert self._execute("exception") == 1 + returncode, stderr = self._execute("exception") + # An unhandled exception in a pipeline should not stop the crawl + assert returncode == 0 + if TWISTED_KEEPS_TRACEBACKS: + assert b'RuntimeError("exception")' in stderr + else: + assert b"RuntimeError: exception" in stderr diff --git a/tests/test_commands.py b/tests/test_commands.py index f63e05628..16af97842 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -670,9 +670,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug("It Works!") - return [] + return + yield """ badspider = """ @@ -680,8 +681,9 @@ import scrapy class BadSpider(scrapy.Spider): name = "bad" - def start_requests(self): + async def start(self): raise Exception("oops!") + yield """ @contextmanager @@ -771,10 +773,10 @@ class MySpider(scrapy.Spider): log = self.get_log("", name="myspider.txt") assert "Unable to load" in log - def test_start_requests_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.py") - assert "start_requests" in log - assert "badspider.py" in log + assert "start" in log + assert "badspider.py" in log, log def test_asyncio_enabled_true(self): log = self.get_log( @@ -846,9 +848,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -862,13 +865,14 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -888,8 +892,9 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): - return [] + async def start(self): + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) @@ -904,9 +909,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) @@ -983,9 +989,10 @@ class MySpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield """ args = ["-a", "foo=42"] log = self.get_log(spider_code, args=args) @@ -1001,9 +1008,9 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): raise unittest.SkipTest("Windows required for .pyw files") return super().setUp() - def test_start_requests_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - assert "start_requests" in log + assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): @@ -1053,9 +1060,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('It works!') - return [] + return + yield """ log = self.get_log(spider_code) assert "[myspider] DEBUG: It works!" in log @@ -1067,9 +1075,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -1083,13 +1092,14 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -1109,8 +1119,9 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): - return [] + async def start(self): + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb961ace2..26b16a1d4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -511,8 +511,9 @@ class TestContractsManager(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(self_): # pylint: disable=no-self-argument - return self.conman.from_spider(self_, self.results) + async def start(self_): # pylint: disable=no-self-argument + for item_or_request in self.conman.from_spider(self_, self.results): + yield item_or_request def parse_first(self, response): self.visited += 1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b7a8a9628..b90706027 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -36,7 +36,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenStartRequestsSpider, + BrokenStartSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -45,14 +45,14 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateStartRequestsSpider, + DuplicateStartSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, - StartRequestsGoodAndBadOutput, - StartRequestsItemSpider, + StartGoodAndBadOutput, + StartItemSpider, ) @@ -165,9 +165,9 @@ class TestCrawl(TestCase): self._assert_retried(log) @defer.inlineCallbacks - def test_start_requests_bug_before_yield(self): + def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -176,9 +176,9 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_bug_yielding(self): + def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -187,14 +187,14 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_items(self): + def test_start_items(self): items = [] def _on_item_scraped(item): items.append(item) with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsItemSpider) + crawler = get_crawler(StartItemSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) @@ -202,11 +202,11 @@ class TestCrawl(TestCase): assert items == [{"name": "test item"}] @defer.inlineCallbacks - def test_start_requests_unsupported_output(self): + def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a - potentially expensive call to itemadapter.is_item, and letting instead - things fail when ItemAdapter is actually used on the corresponding - non-item object.""" + potentially expensive call to itemadapter.is_item(), and letting + instead things fail when ItemAdapter is actually used on the + corresponding non-item object.""" items = [] @@ -214,7 +214,7 @@ class TestCrawl(TestCase): items.append(item) with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler = get_crawler(StartGoodAndBadOutput) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) @@ -223,24 +223,15 @@ class TestCrawl(TestCase): assert not any(isinstance(item, Request) for item in items) @defer.inlineCallbacks - def test_start_requests_laziness(self): + def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenStartRequestsSpider, settings) - yield crawler.crawl(mockserver=self.mockserver) - assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( - 99 - ), crawler.spider.seedsseen - - @defer.inlineCallbacks - def test_start_requests_dupes(self): - settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) assert crawler.spider.visited == 6 - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, @@ -322,10 +313,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # start requests doesn't set Referer header + # start() doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to start request url + # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header @@ -378,27 +369,6 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks - def test_graceful_crawl_error_handling(self): - """ - Test whether errors happening anywhere in Crawler.crawl() are properly - reported (and not somehow swallowed) after a graceful engine shutdown. - The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.start_requests(), - SpiderMiddleware.process_start_requests(), etc. - """ - - class TestError(Exception): - pass - - class FaultySpider(SimpleSpider): - def start_requests(self): - raise TestError - - crawler = get_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - assert not crawler.crawling - @defer.inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { diff --git a/tests/test_crawler.py b/tests/test_crawler.py index efb346dde..7a3d562e5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -153,7 +153,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -233,7 +233,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -313,7 +313,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -393,7 +393,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -580,8 +580,9 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield @pytest.mark.usefixtures("reactor_pytest") diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 408160ccb..8ae160f8a 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -25,7 +25,7 @@ class TestManagerBase(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) self.crawler.engine = self.crawler._create_engine() - return self.crawler.engine.open_spider(self.spider, start_requests=()) + return self.crawler.engine.open_spider(self.spider) def tearDown(self): return self.crawler.engine.close_spider(self.spider) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 4fca9eefb..78c83ea83 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -28,7 +28,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } - def start_requests(self): + async def start(self): self.times = {None: []} slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] diff --git a/tests/test_engine.py b/tests/test_engine.py index 8928e4daf..b60b510b2 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -29,7 +29,7 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request @@ -92,8 +92,9 @@ class MySpider(Spider): class DupeFilterSpider(MySpider): - def start_requests(self): - return (Request(url) for url in self.start_urls) # no dont_filter=True + async def start(self): + for url in self.start_urls: + yield Request(url) # no dont_filter=True class DictItemsSpider(MySpider): @@ -149,7 +150,6 @@ class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): - self.spider = None self.respplug = [] self.reqplug = [] self.reqdropped = [] @@ -190,7 +190,6 @@ class CrawlerRun: self.response_downloaded, signals.response_downloaded ) self.crawler.crawl(start_urls=start_urls) - self.spider = self.crawler.spider self.deferred = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) @@ -296,7 +295,7 @@ class TestEngineBase(unittest.TestCase): assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: assert failure.value.__class__ is ZeroDivisionError - assert spider == run.spider + assert spider == run.crawler.spider assert item["url"] == response.url if "item1.html" in item["url"]: @@ -377,11 +376,14 @@ class TestEngineBase(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - assert {"spider": run.spider} == run.signals_caught[signals.spider_opened] - assert {"spider": run.spider} == run.signals_caught[signals.spider_idle] - assert {"spider": run.spider, "reason": "finished"} == run.signals_caught[ - signals.spider_closed + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] class TestEngine(TestEngineBase): @@ -419,9 +421,10 @@ class TestEngine(TestEngineBase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - assert {"spider": run.spider, "reason": "custom_reason"} == run.signals_caught[ - signals.spider_closed - ] + assert { + "spider": run.crawler.spider, + "reason": "custom_reason", + } == run.signals_caught[signals.spider_closed] @defer.inlineCallbacks def test_close_downloader(self): @@ -471,7 +474,7 @@ class TestEngine(TestEngineBase): finally: timer.cancel() - assert b"Traceback" not in stderr + assert b"Traceback" not in stderr, stderr def test_request_scheduled_signal(caplog): @@ -491,7 +494,13 @@ def test_request_scheduled_signal(caplog): engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() - engine.slot = Slot((), None, Mock(), scheduler) + + async def start(): + return + yield + + engine._start = start() + engine._slot = _Slot(False, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") engine._schedule_request(keep_request) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py new file mode 100644 index 000000000..90af10f0e --- /dev/null +++ b/tests/test_engine_loop.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +from collections import deque +from logging import ERROR +from typing import TYPE_CHECKING + +from testfixtures import LogCapture +from twisted.internet.defer import Deferred +from twisted.trial.unittest import TestCase + +from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .mockserver import MockServer +from .test_scheduler import MemoryScheduler + +if TYPE_CHECKING: + from scrapy.http import Response + + +async def sleep(seconds: float = 0.001) -> None: + from twisted.internet import reactor + + deferred: Deferred[None] = Deferred() + reactor.callLater(seconds, deferred.callback, None) + await maybe_deferred_to_future(deferred) + + +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_sleep(self): + """Neither asynchronous sleeps on Spider.start() nor the equivalent on + the scheduler (returning no requests while also returning True from + the has_pending_requests() method) should cause the spider to miss the + processing of any later requests.""" + seconds = 2 + + class TestSpider(Spider): + name = "test" + + async def start(self): + from twisted.internet import reactor + + yield Request("data:,a") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + + # During this time, the scheduler reports having requests but + # returns None. + await sleep(seconds) + + self.crawler.engine._slot.scheduler.unpause() + + # The scheduler request is processed. + await sleep(seconds) + + yield Request("data:,c") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + + # The last start request is processed during the time until the + # delayed call below, proving that the start iteration can + # finish before a scheduler “sleep” without causing the + # scheduler to finish. + reactor.callLater(seconds, self.crawler.engine._slot.scheduler.unpause) + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_close_during_start_iteration(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + assert self.crawler.engine is not None + await maybe_deferred_to_future(self.crawler.engine.close()) + yield Request("data:,a") + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + + with LogCapture(level=ERROR) as log: + await maybe_deferred_to_future(crawler.crawl()) + + assert not log.records, f"{log.records=}" + finish_reason = crawler.stats.get_value("finish_reason") + assert finish_reason == "shutdown", f"{finish_reason=}" + expected_urls = [] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + +class RequestSendOrderTestCase(TestCase): + seconds = 0.1 # increase if flaky + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) # increase if flaky + + def request(self, num, response_seconds, download_slots, priority=0): + url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") + meta = {"download_slot": str(num % download_slots)} + return Request(url, meta=meta, priority=priority) + + def get_num(self, request_or_response: Request | Response): + return int(request_or_response.url.rsplit("&", maxsplit=1)[1]) + + @deferred_f_from_coro_f + async def _test_request_order( + self, + start_nums, + cb_nums=None, + settings=None, + response_seconds=None, + download_slots=1, + start_fn=None, + parse_fn=None, + ): + cb_nums = cb_nums or [] + settings = settings or {} + response_seconds = response_seconds or self.seconds + + cb_requests = deque( + [self.request(num, response_seconds, download_slots) for num in cb_nums] + ) + + if start_fn is None: + + async def start_fn(spider): + for num in start_nums: + yield self.request(num, response_seconds, download_slots) + + if parse_fn is None: + + def parse_fn(spider, response): + while cb_requests: + yield cb_requests.popleft() + + class TestSpider(Spider): + name = "test" + start = start_fn + parse = parse_fn + + actual_nums = [] + + def track_num(request, spider): + actual_nums.append(self.get_num(request)) + + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_num, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_nums = sorted(start_nums + cb_nums) + assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + + @deferred_f_from_coro_f + async def test_default(self): + """By default, start requests take priority over callback requests and + are sent in order. Priority matters, but given the same priority, a + start request takes precedence.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(4, priority=1), + _request(6), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(5) + yield _request(2, priority=1) + yield _request(3, priority=1) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + @deferred_f_from_coro_f + async def test_lifo_start(self): + """Changing the queues of start requests to LIFO, matching the queues + of non-start requests, does not cause all requests to be stored in the + same queue objects, it only affects the order of start requests.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(4, priority=1), + _request(6), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(5) + yield _request(3, priority=1) + yield _request(2, priority=1) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + @deferred_f_from_coro_f + async def test_shared_queues(self): + """If SCHEDULER_START_*_QUEUE is falsy, start requests and other + requests share the same queue, i.e. start requests are not priorized + over other requests if their priority matches.""" + nums = list(range(1, 14)) + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + # Below, priority 1 requests are sent first, and requests are sent + # in LIFO order. + + for request in ( + _request(7, priority=1), + _request(6, priority=1), + _request(13), + _request(12), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + yield _request(11) + yield _request(10) + yield _request(5, priority=1) + yield _request(4, priority=1) + + for request in ( + _request(3, priority=1), + _request(2, priority=1), + _request(9), + _request(8), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": None, + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + # Examples from the “Start requests” section of the documentation about + # spiders. + + @deferred_f_from_coro_f + async def test_lazy(self): + start_nums = [1, 2, 4] + cb_nums = [3] + response_seconds = self.seconds * 2**1 # increase if flaky + download_slots = 1 + + async def start(spider): + for num in start_nums: + if spider.crawler.engine.needs_backout(): + await spider.crawler.signals.wait_for(signals.scheduler_empty) + request = self.request(num, response_seconds, download_slots) + yield request + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={ + "CONCURRENT_REQUESTS": 1, + }, + response_seconds=response_seconds, + start_fn=start, + ) + ) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 743d9774b..d658d1526 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -69,7 +69,7 @@ class AsyncDefNotAsyncioPipeline: class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index ab6baa5f0..79b53b33b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_start_requests(self, start_requests, spider): - for request in start_requests: + async def process_start(self, start): + async for request in start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_start_requests"] = True + request.cb_kwargs["from_process_start"] = True yield request def process_spider_input(self, response, spider): @@ -62,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] - def start_requests(self): + async def start(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(bool(from_process_response)) self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw( - self, response, from_process_spider_input, from_process_start_requests - ): + def parse_spider_mw(self, response, from_process_spider_input, from_process_start): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_start_requests)) + self.checks.append(bool(from_process_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 1d6992a32..f90293dd3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil import tempfile from abc import ABC, abstractmethod +from collections import deque from typing import Any, NamedTuple import pytest @@ -10,7 +11,7 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider @@ -20,6 +21,38 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +class MemoryScheduler(BaseScheduler): + paused = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.queue = deque( + Request(value) if isinstance(value, str) else value + for value in getattr(self, "queue", []) + ) + + def enqueue_request(self, request: Request) -> bool: + self.queue.append(request) + return True + + def has_pending_requests(self) -> bool: + return self.paused or bool(self.queue) + + def next_request(self) -> Request | None: + if self.paused: + return None + try: + return self.queue.pop() + except IndexError: + return None + + def pause(self) -> None: + self.paused = True + + def unpause(self) -> None: + self.paused = False + + class MockEngine(NamedTuple): downloader: MockDownloader diff --git a/tests/test_signals.py b/tests/test_signals.py index f5075fb60..663e912b7 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,8 +1,9 @@ import pytest from twisted.internet import defer -from twisted.trial import unittest +from twisted.trial.unittest import TestCase from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -10,7 +11,7 @@ from tests.mockserver import MockServer class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} @@ -20,7 +21,21 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class TestAsyncSignal(unittest.TestCase): +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_scheduler_empty(self): + crawler = get_crawler() + calls = [] + + def track_call(): + calls.append(object()) + + crawler.signals.connect(track_call, signals.scheduler_empty) + await maybe_deferred_to_future(crawler.crawl()) + assert len(calls) >= 1 + + +class MockServerTestCase(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_spider.py b/tests/test_spider.py index aaf72390d..b4aa649a3 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,7 @@ import gzip -import inspect import warnings from io import BytesIO -from logging import WARNING +from logging import ERROR, WARNING from pathlib import Path from typing import Any from unittest import mock @@ -27,6 +26,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -45,12 +45,6 @@ class TestSpider(unittest.TestCase): assert spider.name == "example.com" assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison - def test_start_requests(self): - spider = self.spider_class("example.com") - start_requests = spider.start_requests() - assert inspect.isgenerator(start_requests) - assert not list(start_requests) - def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") @@ -152,6 +146,22 @@ class TestSpider(unittest.TestCase): class TestInitSpider(TestSpider): spider_class = InitSpider + @deferred_f_from_coro_f + async def test_start_urls(self): + responses = [] + + class TestSpider(self.spider_class): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + responses.append(response) + + crawler = get_crawler(TestSpider) + await maybe_deferred_to_future(crawler.crawl()) + assert len(responses) == 1 + assert responses[0].url == "data:," + class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider @@ -454,12 +464,17 @@ class TestCrawlSpider(TestSpider): assert hasattr(spider, "_follow_links") assert not spider._follow_links + @inlineCallbacks def test_start_url(self): - spider = self.spider_class("example.com") - spider.start_url = "https://www.example.com" + class TestSpider(self.spider_class): + name = "test" + start_url = "https://www.example.com" - with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"): - list(spider.start_requests()) + crawler = get_crawler(TestSpider) + with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: + yield crawler.crawl() + assert "Error while reading start items and requests" in str(log) + assert "did you miss an 's'?" in str(log) class TestSitemapSpider(TestSpider): @@ -776,6 +791,24 @@ Sitemap: /sitemap-relative-url.xml ), ) + @deferred_f_from_coro_f + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + with warnings.catch_warnings(): + warnings.simplefilter("error") + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap + class TestDeprecation: def test_crawl_spider(self): diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py new file mode 100644 index 000000000..1815aad76 --- /dev/null +++ b/tests/test_spider_start.py @@ -0,0 +1,186 @@ +import warnings +from asyncio import sleep + +import pytest +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase + +from scrapy import Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .utils import twisted_sleep + +SLEEP_SECONDS = 0.1 + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} + + +class MainTestCase(TestCase): + async def _test_spider(self, spider, expected_items=None): + actual_items = [] + expected_items = [] if expected_items is None else expected_items + + def track_item(item, response, spider): + actual_items.append(item) + + crawler = get_crawler(spider) + crawler.signals.connect(track_item, signals.item_scraped) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items + + @deferred_f_from_coro_f + async def test_start_urls(self): + class TestSpider(Spider): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_deprecated(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_A + + with pytest.warns(ScrapyDeprecationWarning): + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_deprecated_subclass(self): + class BaseSpider(Spider): + def start_requests(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + # The warning must be about the base class and not the subclass. + with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_universal(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + def start_requests(self): + yield ITEM_B + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_universal_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + def start_requests(self): + yield ITEM_B + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_deprecated_super(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + for item_or_request in super().start_requests(): + yield item_or_request + + with pytest.warns( + ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead" + ) as messages: + await self._test_spider(TestSpider, []) + assert messages[0].filename.endswith("test_spider_start.py") + + async def _test_start(self, start_, expected_items=None): + class TestSpider(Spider): + name = "test" + start = start_ + + await self._test_spider(TestSpider, expected_items) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_delayed(self): + async def start(spider): + await sleep(SLEEP_SECONDS) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_twisted_delayed(self): + async def start(spider): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + # Exceptions + + @deferred_f_from_coro_f + async def test_deprecated_non_generator_exception(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + raise RuntimeError + + with ( + LogCapture() as log, + pytest.warns( + ScrapyDeprecationWarning, + match=r"defines the deprecated start_requests\(\) method", + ), + ): + await self._test_spider(TestSpider, []) + + assert "in start_requests\n raise RuntimeError" in str(log) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 1d671134e..db46be7dd 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from inspect import isasyncgen from typing import Any from unittest import mock @@ -111,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_start_requests. + Should work for process_spider_output and, when it's supported, process_start. """ ITEM_TYPE: type | tuple @@ -200,7 +201,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware: yield {"foo": 3} -class ProcessSpiderExceptionAsyncIterableMiddleware: +class ProcessSpiderExceptionAsyncIteratorMiddleware: async def process_spider_exception(self, response, exception, spider): yield {"foo": 1} d = defer.Deferred() @@ -319,37 +320,43 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): ) -class ProcessStartRequestsSimpleMiddleware: - def process_start_requests(self, start_requests, spider): - yield from start_requests +class ProcessStartSimpleMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request -class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): - """process_start_requests tests for simple start_requests""" +class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): + """process_start tests for simple start""" ITEM_TYPE = (Request, dict) - MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + MW_SIMPLE = ProcessStartSimpleMiddleware - def _start_requests(self): - for i in range(2): - yield Request(f"https://example.com/{i}", dont_filter=True) - yield {"name": "test item"} + async def _get_processed_start(self, *mw_classes): + class TestSpider(Spider): + name = "test" - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): - setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + async def start(self): + for i in range(2): + yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} + + setting = self._construct_mw_setting(*mw_classes) self.crawler = get_crawler( - Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) - self.spider = self.crawler._create_spider("foo") + self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - start_requests = iter(self._start_requests()) - results = yield self.mwman.process_start_requests(start_requests, self.spider) - return results + return await self.mwman.process_start(self.spider) - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + start = await self._get_processed_start(self.MW_SIMPLE) + assert isasyncgen(start) + start_list = await collect_asyncgen(start) + assert len(start_list) == self.RESULT_COUNT + assert isinstance(start_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: @@ -507,7 +514,7 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware - MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware def _scrape_func(self, *args, **kwargs): 1 / 0 diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index 46be879f3..77d055d50 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -27,16 +27,19 @@ def test_trivial(crawler): assert mw.crawler is crawler test_req = Request("data:,") spider_output = [test_req, {"foo": "bar"}] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert processed == [test_req, {"foo": "bar"}] + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert processed == [test_req, {"foo": "bar"}] def test_processed_request(crawler): class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if request.url == "data:2,": return None @@ -49,20 +52,23 @@ def test_processed_request(crawler): test_req2 = Request("data:2,") test_req3 = Request("data:3,") spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert len(processed) == 3 - assert isinstance(processed[0], Request) - assert processed[0].url == "data:1," - assert processed[1] == {"foo": "bar"} - assert isinstance(processed[2], Request) - assert processed[2].url == "data:30," + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," def test_processed_item(crawler): class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: return None if item["foo"] == 3: @@ -72,16 +78,19 @@ def test_processed_item(crawler): mw = ProcessItemSpiderMiddleware.from_crawler(crawler) test_req = Request("data:,") spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert processed == [{"foo": 1}, test_req, {"foo": 30}] + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert processed == [{"foo": 1}, test_req, {"foo": 30}] def test_processed_both(crawler): class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if request.url == "data:2,": return None @@ -89,7 +98,7 @@ def test_processed_both(crawler): return Request("data:30,") return request - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: return None if item["foo"] == 3: @@ -108,13 +117,16 @@ def test_processed_both(crawler): {"foo": 3}, test_req3, ] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert len(processed) == 4 - assert isinstance(processed[0], Request) - assert processed[0].url == "data:1," - assert processed[1] == {"foo": 1} - assert processed[2] == {"foo": 30} - assert isinstance(processed[3], Request) - assert processed[3].url == "data:30," + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e306579fa..fd2fc3581 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -30,7 +30,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - def start_requests(self): + async def start(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 6e26a85ea..20efac543 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -36,7 +36,7 @@ class RecoverySpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -73,7 +73,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - def start_requests(self): + async def start(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -83,7 +83,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - def start_requests(self): + async def start(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -103,7 +103,7 @@ class GeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -140,7 +140,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -215,7 +215,7 @@ class GeneratorOutputChainSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -287,8 +287,8 @@ class NotGeneratorOutputChainSpider(Spider): }, } - def start_requests(self): - return [Request(self.mockserver.url("/status?n=200"))] + async def start(self): + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): return [ diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py new file mode 100644 index 000000000..725833a49 --- /dev/null +++ b/tests/test_spidermiddleware_process_start.py @@ -0,0 +1,352 @@ +import warnings +from asyncio import sleep + +import pytest +from twisted.trial.unittest import TestCase + +from scrapy import Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.test_spider_start import SLEEP_SECONDS + +from .utils import twisted_sleep + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} +ITEM_C = {"id": "c"} +ITEM_D = {"id": "d"} + + +class AsyncioSleepSpiderMiddleware: + async def process_start(self, start): + await sleep(SLEEP_SECONDS) + async for item_or_request in start: + yield item_or_request + + +class NoOpSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + +class TwistedSleepSpiderMiddleware: + async def process_start(self, start): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + async for item_or_request in start: + yield item_or_request + + +class UniversalSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + def process_start_requests(self, start_requests, spider): + raise NotImplementedError + + +# Spiders and spider middlewares for MainTestCase._test_wrap + + +class ModernWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + +class ModernWrapSpiderSubclass(ModernWrapSpider): + name = "test" + + +class UniversalWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + def start_requests(self): + yield ITEM_D + + +class DeprecatedWrapSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_B + + +class ModernWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + +class UniversalWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + def process_start_requests(self, start, spider): + yield ITEM_A + yield from start + yield ITEM_C + + +class DeprecatedWrapSpiderMiddleware: + def process_start_requests(self, start, spider): + yield ITEM_A + yield from start + yield ITEM_C + + +class MainTestCase(TestCase): + async def _test(self, spider_middlewares, spider_cls, expected_items): + actual_items = [] + + def track_item(item, response, spider): + actual_items.append(item) + + settings = { + "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)}, + } + crawler = get_crawler(spider_cls, settings_dict=settings) + crawler.signals.connect(track_item, signals.item_scraped) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" + + async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] + await self._test([spider_middleware], spider_cls, expected_items) + + async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] + await self._test([smw1, smw2], spider_cls, expected_items) + + @deferred_f_from_coro_f + async def test_modern_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider_subclass(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, + match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders", + ), + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_D, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_modern_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_universal_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_deprecated_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + async def _test_sleep(self, spider_middlewares): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + await self._test(spider_middlewares, TestSpider, [ITEM_A]) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_sleep_single(self): + await self._test_sleep([AsyncioSleepSpiderMiddleware]) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) + + @deferred_f_from_coro_f + async def test_twisted_sleep_single(self): + await self._test_sleep([TwistedSleepSpiderMiddleware]) + + @deferred_f_from_coro_f + async def test_twisted_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py new file mode 100644 index 000000000..295b10ea8 --- /dev/null +++ b/tests/test_spidermiddleware_start.py @@ -0,0 +1,44 @@ +from twisted.trial.unittest import TestCase + +from scrapy.http import Request +from scrapy.spidermiddlewares.start import StartSpiderMiddleware +from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler + + +class TestMiddleware(TestCase): + @deferred_f_from_coro_f + async def test_async(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + async def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + async for request in mw.process_start(start()) + ] + assert result == [True, True, False, "foo"] + + @deferred_f_from_coro_f + async def test_sync(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + for request in mw.process_start_requests(start(), Spider("test")) + ] + assert result == [True, True, False, "foo"] diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e69de29bb..e5e56f414 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -0,0 +1,9 @@ +from twisted.internet.defer import Deferred + + +def twisted_sleep(seconds): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(seconds, d.callback, None) + return d diff --git a/tox.ini b/tox.ini index e63e44189..92cfc3794 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] -basepython = python3 +basepython = python3.9 deps = mypy==1.14.0 typing-extensions==4.12.2