From ea3e6d2e42d2892066d0fe53ebf9fce28ebc4e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 19 Mar 2025 19:42:13 +0100 Subject: [PATCH 01/45] =?UTF-8?q?yield=5Fseeds,=20process=5Fseeds=20?= =?UTF-8?q?=E2=86=92=20start,=20process=5Fstart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/contributing.rst | 4 +- docs/intro/tutorial.rst | 14 ++--- docs/news.rst | 16 ++--- docs/topics/coroutines.rst | 6 +- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 4 +- docs/topics/signals.rst | 6 +- docs/topics/spider-middleware.rst | 10 ++-- docs/topics/spiders.rst | 16 ++--- extras/qpsclient.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 4 +- scrapy/commands/fetch.py | 4 +- scrapy/commands/parse.py | 4 +- scrapy/core/engine.py | 2 +- scrapy/core/scheduler.py | 2 +- scrapy/core/spidermw.py | 58 +++++++++---------- scrapy/http/request/__init__.py | 2 +- scrapy/logformatter.py | 2 +- scrapy/spiders/__init__.py | 12 ++-- scrapy/spiders/init.py | 2 +- scrapy/spiders/sitemap.py | 2 +- .../project/module/middlewares.py.tmpl | 4 +- tests/CrawlerProcess/args_settings.py | 2 +- tests/CrawlerProcess/asyncio_custom_loop.py | 2 +- .../asyncio_enabled_no_reactor.py | 2 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 2 +- .../asyncio_enabled_reactor_different_loop.py | 2 +- .../asyncio_enabled_reactor_same_loop.py | 2 +- .../caching_hostname_resolver.py | 2 +- tests/CrawlerProcess/multi.py | 2 +- tests/CrawlerProcess/reactor_default.py | 2 +- .../reactor_default_twisted_reactor_select.py | 2 +- tests/CrawlerProcess/reactor_select.py | 2 +- ..._select_subclass_twisted_reactor_select.py | 2 +- .../reactor_select_twisted_reactor_select.py | 2 +- tests/CrawlerProcess/simple.py | 2 +- tests/CrawlerRunner/change_reactor.py | 2 +- tests/CrawlerRunner/ip_address.py | 2 +- tests/spiders.py | 24 ++++---- tests/test_commands.py | 30 +++++----- tests/test_contracts.py | 2 +- tests/test_crawl.py | 22 +++---- tests/test_crawler.py | 10 ++-- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine.py | 2 +- tests/test_engine_seeding.py | 4 +- tests/test_pipelines.py | 2 +- tests/test_request_cb_kwargs.py | 12 ++-- tests/test_signals.py | 2 +- ...er_yield_seeds.py => test_spider_start.py} | 24 ++++---- tests/test_spidermiddleware.py | 12 ++-- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 14 ++--- ...=> test_spidermiddleware_process_start.py} | 20 +++---- 55 files changed, 198 insertions(+), 198 deletions(-) rename tests/{test_spider_yield_seeds.py => test_spider_start.py} (87%) rename tests/{test_spidermiddleware_process_seeds.py => test_spidermiddleware_process_start.py} (93%) diff --git a/docs/contributing.rst b/docs/contributing.rst index bb197b428..34e92d8f0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -228,8 +228,8 @@ with a name of the branch you want to create locally). See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. -E.g. For bug #411: "Scrapy hangs if an exception raises in yield_seeds" prefer -"Fix hanging when exception occurs in yield_seeds (#411)" instead of "Fix for +E.g. For bug #411: "Scrapy hangs if an exception raises in start" prefer +"Fix hanging when exception occurs in start (#411)" instead of "Fix for #411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8f74f76af..71b420be6 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - async def yield_seeds(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,7 +116,7 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.yield_seeds`: must be an asynchronous generator that +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that yields requests (and, optionally, items) for the spider to start crawling. Subsequent requests will be generated successively from these initial requests. @@ -165,20 +165,20 @@ What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Scrapy sends the first :class:`scrapy.Request ` objects yielded -by the :meth:`~scrapy.Spider.yield_seeds` spider method. Upon receiving a +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a response for each one, Scrapy calls the callback method associated with the request (in this case, the ``parse`` method) with a :class:`~scrapy.http.Response` object. -A shortcut to the ``yield_seeds`` method +A shortcut to the ``start`` method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.yield_seeds` method that yields +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields :class:`~scrapy.Request` objects from URLs, you can define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation of -:meth:`~scrapy.Spider.yield_seeds` to create the initial requests for your +:meth:`~scrapy.Spider.start` to create the initial requests for your spider. .. code-block:: python @@ -795,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - async def yield_seeds(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 93e003399..de7506e42 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,20 +10,20 @@ Scrapy VERSION (unreleased) Highlights: -- Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.yield_seeds` +- Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.start` (async) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - In ``scrapy.core.spidermw.SpiderMiddlewareManager``, - ``process_start_requests()`` has been replaced by ``process_seeds()``. + ``process_start_requests()`` has been replaced by ``process_start()``. - In ``scrapy.core.engine.ExecutionEngine``: - The second parameter of ``open_spider()``, ``start_requests``, has been removed. The starting requests are determined by the ``spider`` - parameter instead (see :meth:`~scrapy.Spider.yield_seeds`). + parameter instead (see :meth:`~scrapy.Spider.start`). - The ``slot`` attribute has been renamed to ``_slot`` and should not be used. @@ -37,7 +37,7 @@ Deprecations ~~~~~~~~~~~~ - The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, - use :meth:`~scrapy.Spider.yield_seeds` instead, or both to maintain support + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for lower Scrapy versions. (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, @@ -45,7 +45,7 @@ Deprecations - The ``process_start_requests()`` method of :ref:`spider middlewares ` is deprecated, use - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` instead, or + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, or both to maintain support for lower Scrapy versions. (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, @@ -55,8 +55,8 @@ New features ~~~~~~~~~~~~ - You can now yield the start requests and items of a spider from the - :meth:`~scrapy.Spider.yield_seeds` spider method and from the - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` spider + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider middleware method, both asynchronous generators. This makes it possible to use asynchronous code to generate those start @@ -68,7 +68,7 @@ New features Bug fixes ~~~~~~~~~ -- Yielding a start item (i.e. from :meth:`~scrapy.Spider.yield_seeds` or an +- Yielding a start item (i.e. from :meth:`~scrapy.Spider.start` or an equivalent) no longer delays the next iteration of starting requests and items by up to 5 seconds. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index bed13f483..2a7a0c7c7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -18,7 +18,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- The :meth:`~scrapy.spiders.Spider.yield_seeds` spider method. +- The :meth:`~scrapy.spiders.Spider.start` spider method. .. versionadded: VERSION @@ -54,7 +54,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 -- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` method +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of :ref:`spider middlewares `. .. versionadded:: VERSION @@ -157,7 +157,7 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in - :meth:`~scrapy.spiders.Spider.yield_seeds`, callbacks, pipelines and + :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a81832023..2032fc5cf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -353,7 +353,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - async def yield_seeds(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5bc1ccc7d..67de4f5f1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1950,7 +1950,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - async def yield_seeds(self): + async def start(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1979,7 +1979,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - async def yield_seeds(self): + async def start(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 782ac1e26..091a4c000 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -160,7 +160,7 @@ item_scraped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.yield_seeds`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped @@ -181,7 +181,7 @@ item_dropped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.yield_seeds`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a @@ -205,7 +205,7 @@ item_error :param response: the response being processed when the exception was raised, or ``None`` if it was yielded from - :meth:`~scrapy.Spider.yield_seeds`. + :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 55330da9f..4632644e6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,21 +70,21 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_seeds(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] + .. method:: process_start(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] :async: - Iterate over the output of :meth:`~scrapy.Spider.yield_seeds` or that - of the :meth:`process_seeds` method of an earlier spider middleware, + Iterate over the output of :meth:`~scrapy.Spider.start` or that + of the :meth:`process_start` method of an earlier spider middleware, overriding it. For example: .. code-block:: python - async def process_seeds(self, seeds): + async def process_start(self, seeds): async for seed in seeds: yield seed You may yield :class:`~scrapy.Request` or :ref:`item ` - objects, same as :meth:`~scrapy.Spider.yield_seeds`, from *seeds* or + objects, same as :meth:`~scrapy.Spider.start`, from *seeds* or not. To write spider middlewares that work on Scrapy versions lower than diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 88eed477a..a05a4b14d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,7 +17,7 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by iterating the - :meth:`~scrapy.Spider.yield_seeds` method, which by default yields a + :meth:`~scrapy.Spider.start` method, which by default yields a :class:`~scrapy.Request` object for each URL in the :attr:`~scrapy.Spider.start_urls` spider attribute, with the :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` @@ -137,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`yield_seeds` method, handlers of the + :meth:`start` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -189,7 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. automethod:: yield_seeds + .. automethod:: start .. method:: parse(response) @@ -261,7 +261,7 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.yield_seeds` directly; +Instead of :attr:`~.start_urls` you can use :meth:`~.start` directly; to give data more structure you can use :class:`~scrapy.Item` objects: .. skip: next @@ -275,7 +275,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - async def yield_seeds(self): + async def start(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -329,7 +329,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - async def yield_seeds(self): + async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") If you are :ref:`running Scrapy from a script `, you can @@ -893,8 +893,8 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - async def yield_seeds(self): - async for seed in super().yield_seeds(): + async def start(self): + async for seed in super().start(): yield seed for url in self.other_urls: yield Request(url, self.parse_other) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index df1bf8767..f95c6010f 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,7 +34,7 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) - async def yield_seeds(self): + async def start(self): for seed in self.start_requests(): yield seed diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 5ed09c77c..4cd5d640a 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -59,7 +59,7 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" yield scrapy.Request(url, dont_filter=True) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 24dfc0106..59c5ab9b5 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -80,7 +80,7 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - async def yield_seeds(self): + async def start(self): for request in conman.from_spider(self, self._result): yield request @@ -89,7 +89,7 @@ class Command(ScrapyCommand): spidercls = spider_loader.load(spidername) spidercls._result = result # type: ignore[assignment,attr-defined,method-assign,return-value] - spidercls.yield_seeds = yield_seeds # type: ignore[assignment,method-assign,return-value] + spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d1b0974ec..77c318958 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -90,11 +90,11 @@ class Command(ScrapyCommand): else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - async def yield_seeds(self): + async def start(self): yield self._request spidercls._request = request # type: ignore[assignment,attr-defined] - spidercls.yield_seeds = yield_seeds # type: ignore[method-assign,attr-defined] + spidercls.start = start # type: ignore[method-assign,attr-defined] self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 810824c7e..a43f55137 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -258,11 +258,11 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - async def yield_seeds(spider: Spider) -> AsyncIterable[Any]: + async def start(spider: Spider) -> AsyncIterable[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.yield_seeds = yield_seeds # type: ignore[assignment,method-assign] + self.spidercls.start = start # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index a37dc6b13..23cddf481 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -380,7 +380,7 @@ class ExecutionEngine: logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._start_next_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - self._seeds = yield self.scraper.spidermw.process_seeds(spider) + self._seeds = yield self.scraper.spidermw.process_start(spider) self._slot = _Slot(close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open") and (d := scheduler.open(spider)): diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 287b15c4b..3c1411a18 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -56,7 +56,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): The original sources of said requests are: - * Spider: ``yield_seeds`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``start`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0affa4534..9a292cae9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -67,7 +67,7 @@ class SpiderMiddlewareManager(MiddlewareManager): middleware for middleware in middlewares if hasattr(middleware, "process_start_requests") - and not hasattr(middleware, "process_seeds") + and not hasattr(middleware, "process_start") ] self._use_start_requests = bool(deprecated_middlewares) if self._use_start_requests: @@ -80,16 +80,16 @@ class SpiderMiddlewareManager(MiddlewareManager): f"through their parent classes, define the deprecated " f"process_start_requests() method: " f"{deprecated_middleware_list}. process_start_requests() has " - f"been deprecated in favor of a new method, process_seeds(), " + f"been deprecated in favor of a new method, process_start(), " f"to support asynchronous code execution. " f"process_start_requests() will stop being called in a future " f"version of Scrapy. If you use Scrapy VERSION or higher " f"only, replace process_start_requests() with " - f"process_seeds(); note that process_seeds() is a coroutine " + f"process_start(); note that process_start() is a coroutine " f"(async def). If you need to maintain compatibility with " f"lower Scrapy versions, when defining " f"process_start_requests() in a spider middleware class, " - f"define process_seeds() as well. See the release notes of " + f"define process_start() as well. See the release notes of " f"Scrapy VERSION for details: " f"https://docs.scrapy.org/en/VERSION/news.html", ScrapyDeprecationWarning, @@ -104,8 +104,8 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods["process_start_requests"].appendleft( mw.process_start_requests ) - elif hasattr(mw, "process_seeds"): - self.methods["process_seeds"].appendleft(mw.process_seeds) + elif hasattr(mw, "process_start"): + self.methods["process_start"].appendleft(mw.process_start) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) process_spider_exception = getattr(mw, "process_spider_exception", None) @@ -369,7 +369,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return dfd2 @inlineCallbacks - def process_seeds( + def process_start( self, spider: Spider ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]: self._check_deprecated_start_requests_use(spider) @@ -381,12 +381,12 @@ class SpiderMiddlewareManager(MiddlewareManager): seeds = as_async_generator(sync_seeds) else: seeds = yield self._iter_seeds(spider) - seeds = yield self._process_chain("process_seeds", seeds) + seeds = yield self._process_chain("process_start", seeds) return seeds def _check_deprecated_start_requests_use(self, spider: Spider): start_requests_cls = None - yield_seeds_cls = None + start_cls = None spidercls = spider.__class__ mro = spidercls.__mro__ @@ -394,19 +394,19 @@ class SpiderMiddlewareManager(MiddlewareManager): cls_dict = cls.__dict__ if start_requests_cls is None and "start_requests" in cls_dict: start_requests_cls = cls - if yield_seeds_cls is None and "yield_seeds" in cls_dict: - yield_seeds_cls = cls - if start_requests_cls is not None and yield_seeds_cls is not None: + if start_cls is None and "start" in cls_dict: + start_cls = cls + if start_requests_cls is not None and start_cls is not None: break - # Spider defines both, start_requests and yield_seeds. + # Spider defines both, start_requests and start. assert start_requests_cls is not None - assert yield_seeds_cls is not None + assert start_cls is not None if ( start_requests_cls is not Spider - and yield_seeds_cls is not start_requests_cls - and mro.index(start_requests_cls) < mro.index(yield_seeds_cls) + and start_cls is not start_requests_cls + and mro.index(start_requests_cls) < mro.index(start_cls) ): src = global_object_name(start_requests_cls) if start_requests_cls is not spidercls: @@ -414,15 +414,15 @@ class SpiderMiddlewareManager(MiddlewareManager): warn( f"{src} defines the deprecated start_requests() method. " f"start_requests() has been deprecated in favor of a new " - f"method, yield_seeds(), to support asynchronous code " + f"method, start(), to support asynchronous code " f"execution. start_requests() will stop being called in a " f"future version of Scrapy. If you use Scrapy VERSION or " - f"higher only, replace start_requests() with yield_seeds(); " - f"note that yield_seeds() is a coroutine (async def). If you " + f"higher only, replace start_requests() with start(); " + f"note that start() is a coroutine (async def). If you " f"need to maintain compatibility with lower Scrapy versions, " f"when overriding start_requests() in a spider class, " - f"override yield_seeds() as well; you can use super() to " - f"reuse the inherited yield_seeds() implementation without " + f"override start() as well; you can use super() to " + f"reuse the inherited start() implementation without " f"copy-pasting. See the release notes of Scrapy VERSION for " f"details: https://docs.scrapy.org/en/VERSION/news.html", ScrapyDeprecationWarning, @@ -430,33 +430,33 @@ class SpiderMiddlewareManager(MiddlewareManager): if ( self._use_start_requests - and yield_seeds_cls is not Spider - and start_requests_cls is not yield_seeds_cls - and mro.index(yield_seeds_cls) < mro.index(start_requests_cls) + and start_cls is not Spider + and start_requests_cls is not start_cls + and mro.index(start_cls) < mro.index(start_requests_cls) ): - src = global_object_name(yield_seeds_cls) - if yield_seeds_cls is not spidercls: + src = global_object_name(start_cls) + if start_cls is not spidercls: src += f" (inherited by {global_object_name(spidercls)})" raise ValueError( f"{src} does not define the deprecated start_requests() " f"method. However, one or more of your enabled spider " f"middlewares (reported in an earlier deprecation warning) " f"define the process_start_requests() method, and not the " - f"process_seeds() method, making them only compatible with " + f"process_start() method, making them only compatible with " f"(deprecated) spiders that define the start_requests() " f"method. To solve this issue, disable the offending spider " f"middlewares, upgrade them as described in that earlier " f"deprecation warning, or make your spider compatible with " f"deprecated spider middlewares (and earlier Scrapy versions) " f"by defining a sync start_requests() method that works " - f"similarly to its existing yield_seeds() method. See the " + f"similarly to its existing start() method. See the " f"release notes of Scrapy VERSION for details: " f"https://docs.scrapy.org/en/VERSION/news.html" ) @staticmethod def _iter_seeds(spider: Spider): - fn = spider.yield_seeds + fn = spider.start if isasyncgenfunction(fn): return fn().__aiter__() assert iscoroutinefunction(fn) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0d2b3aa99..d72c72ecd 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -191,7 +191,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.yield_seeds`. + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 6315b7adc..4f08918ae 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -98,7 +98,7 @@ class LogFormatter: """Logs a message when an item is scraped by a spider.""" src: Any if response is None: - src = f"{global_object_name(spider.__class__)}.yield_seeds" + src = f"{global_object_name(spider.__class__)}.start" elif isinstance(response, Failure): src = response.getErrorMessage() else: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 5f2ac1a26..eb88f62a4 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: class Spider(object_ref): """Base class that any spider must subclass. - It provides a default :meth:`yield_seeds` implementation that sends + It provides a default :meth:`start` implementation that sends requests based on the :attr:`start_urls` class attribute and calls the :meth:`parse` method for each response. """ @@ -39,7 +39,7 @@ class Spider(object_ref): name: str custom_settings: dict[_SettingsKeyT, Any] | None = None - #: Seed URLs. See :meth:`yield_seeds`. + #: Seed URLs. See :meth:`start`. start_urls: list[str] def __init__(self, name: str | None = None, **kwargs: Any): @@ -78,7 +78,7 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. .. versionadded:: VERSION @@ -93,7 +93,7 @@ class Spider(object_ref): class MySpider(Spider): name = "myspider" - async def yield_seeds(self): + async def start(self): yield Request("https://toscrape.com/") The default implementation reads URLs from :attr:`start_urls` and @@ -102,7 +102,7 @@ class Spider(object_ref): .. code-block:: python - async def yield_seeds(self): + async def start(self): for url in self.start_urls: yield Request(url, dont_filter=True) @@ -110,7 +110,7 @@ class Spider(object_ref): .. code-block:: python - async def yield_seeds(self): + async def start(self): yield {"foo": "bar"} To write spiders that work on Scrapy versions lower than VERSION, diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index c1d38854b..ee0058880 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -29,7 +29,7 @@ class InitSpider(Spider): stacklevel=2, ) - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: for seed in self.start_requests(): yield seed diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index bc004f663..c01b54b4a 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -53,7 +53,7 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: for seed in self.start_requests(): yield seed diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8b8ab927d..59a3dca21 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,8 +43,8 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - async def process_seeds(self, seeds): - # Called with the seeds from the spider yield_seeds() method or with + async def process_start(self, seeds): + # Called with the seeds from the spider start() method or with # the output of the maching method of an earlier spider middleware. async for seed in seeds: yield seed diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index 6076211ec..c8a3d0a5b 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,7 +13,7 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - async def yield_seeds(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") return yield diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 13c6fff85..bd78a0de7 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 950bdb006..6bb6fb3c6 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -12,7 +12,7 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index a80a1180a..f3dab12fe 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -38,7 +38,7 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 59feced94..d8c467f40 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -15,7 +15,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index f297fd0d7..e7d3ca9cc 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -16,7 +16,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 26343da6a..53d427061 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - async def yield_seeds(self): + async def start(self): yield scrapy.Request(self.url) def parse(self, response): diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 65f5e033f..0058896b5 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index a221764d2..8f59c035c 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -8,7 +8,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index a0aff999b..9901dd634 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -8,7 +8,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 6ac1043b5..53941568a 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -10,7 +10,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index f7352af57..5739d77ae 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,7 +17,7 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index 1071e453d..c488f7526 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,7 +9,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 3773092b0..9e4ad70d9 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index bdc217fde..6c0102241 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -10,7 +10,7 @@ class NoRequestsSpider(Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 892fab731..5e2184afb 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -32,7 +32,7 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - async def yield_seeds(self): + async def start(self): yield Request(self.url) def parse(self, response): diff --git a/tests/spiders.py b/tests/spiders.py index 5075a795b..d7fffd001 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -68,7 +68,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - async def yield_seeds(self): + async def start(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -105,7 +105,7 @@ class LogSpider(MetaSpider): class SlowSpider(DelaySpider): name = "slow" - async def yield_seeds(self): + async def start(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -255,7 +255,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - async def yield_seeds(self): + async def start(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -327,7 +327,7 @@ class BrokenYieldSeedsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - async def yield_seeds(self): + async def start(self): if self.fail_before_yield: 1 / 0 @@ -346,12 +346,12 @@ class BrokenYieldSeedsSpider(FollowAllSpider): class YieldSeedsItemSpider(FollowAllSpider): - async def yield_seeds(self): + async def start(self): yield {"name": "test item"} class YieldSeedsGoodAndBadOutput(FollowAllSpider): - async def yield_seeds(self): + async def start(self): yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -363,7 +363,7 @@ class SingleRequestSpider(MetaSpider): callback_func = None errback_func = None - async def yield_seeds(self): + async def start(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -390,7 +390,7 @@ class DuplicateYieldSeedsSpider(MockServerSpider): distinct_urls = 2 dupe_factor = 3 - async def yield_seeds(self): + async def start(self): for i in range(self.distinct_urls): for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") @@ -415,7 +415,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - async def yield_seeds(self): + async def start(self): test_body = b""" Page title<title></head> @@ -469,7 +469,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - async def yield_seeds(self): + async def start(self): test_body = b""" <html> <head><title>Page title<title></head> @@ -514,7 +514,7 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - async def yield_seeds(self): + async def start(self): body = b"a" * self.full_response_length url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=body, errback=self.errback) @@ -543,7 +543,7 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status"), errback=self.errback) def parse(self, response): diff --git a/tests/test_commands.py b/tests/test_commands.py index 84ac8b125..7d399d474 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -670,7 +670,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug("It Works!") return yield @@ -681,7 +681,7 @@ import scrapy class BadSpider(scrapy.Spider): name = "bad" - async def yield_seeds(self): + async def start(self): raise Exception("oops!") """ @@ -772,9 +772,9 @@ class MySpider(scrapy.Spider): log = self.get_log("", name="myspider.txt") assert "Unable to load" in log - def test_yield_seeds_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.py") - assert "yield_seeds" in log + assert "start" in log assert "badspider.py" in log def test_asyncio_enabled_true(self): @@ -847,7 +847,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return yield @@ -864,7 +864,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) @@ -891,7 +891,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): return yield """ @@ -908,7 +908,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return yield @@ -988,7 +988,7 @@ class MySpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - async def yield_seeds(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") return yield @@ -1007,9 +1007,9 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): raise unittest.SkipTest("Windows required for .pyw files") return super().setUp() - def test_yield_seeds_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - assert "yield_seeds" in log + assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): @@ -1059,7 +1059,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('It works!') return yield @@ -1074,7 +1074,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return yield @@ -1091,7 +1091,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) @@ -1118,7 +1118,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): return yield """ diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d06e186cf..00bfe0ef9 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -511,7 +511,7 @@ class TestContractsManager(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - async def yield_seeds(self_): # pylint: disable=no-self-argument + async def start(self_): # pylint: disable=no-self-argument for seed in self.conman.from_spider(self_, self.results): yield seed diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 765d2c0f6..85d016e44 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -163,7 +163,7 @@ class TestCrawl(TestCase): self._assert_retried(log) @defer.inlineCallbacks - def test_yield_seeds_bug_before_yield(self): + def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenYieldSeedsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) @@ -174,7 +174,7 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_yield_seeds_bug_yielding(self): + def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenYieldSeedsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) @@ -185,7 +185,7 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_yield_seeds_items(self): + def test_start_items(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(YieldSeedsItemSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -193,7 +193,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 @defer.inlineCallbacks - def test_yield_seeds_unsupported_output(self): + def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item, and letting instead things fail when ItemAdapter is actually used on the corresponding @@ -205,7 +205,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 @defer.inlineCallbacks - def test_yield_seeds_laziness(self): + def test_start_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(BrokenYieldSeedsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) @@ -214,7 +214,7 @@ class TestCrawl(TestCase): ), crawler.spider.seedsseen @defer.inlineCallbacks - def test_yield_seeds_dupes(self): + def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateYieldSeedsSpider, settings) yield crawler.crawl( @@ -304,10 +304,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # test_yield_seeds doesn't set Referer header + # test_start doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to test_yield_seeds url + # following request sets Referer to test_start url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header @@ -366,15 +366,15 @@ with multiples lines Test whether errors happening anywhere in Crawler.crawl() are properly reported (and not somehow swallowed) after a graceful engine shutdown. The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.test_yield_seeds(), - SpiderMiddleware.process_test_yield_seeds(), etc. + spiders/middlewares/etc., e.g. raised in Spider.test_start(), + SpiderMiddleware.process_test_start(), etc. """ class TestError(Exception): pass class FaultySpider(SimpleSpider): - async def yield_seeds(self): + async def start(self): raise TestError crawler = get_crawler(FaultySpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index fa1c90880..42950ea94 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -153,7 +153,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -233,7 +233,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -313,7 +313,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -393,7 +393,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -580,7 +580,7 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index d485087ae..78c83ea83 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -28,7 +28,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } - async def yield_seeds(self): + async def start(self): self.times = {None: []} slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] diff --git a/tests/test_engine.py b/tests/test_engine.py index 223e89326..306a0e775 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -92,7 +92,7 @@ class MySpider(Spider): class DupeFilterSpider(MySpider): - async def yield_seeds(self): + async def start(self): for url in self.start_urls: yield Request(url) # no dont_filter=True diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index 93e4e701d..03a5ac64a 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -10,7 +10,7 @@ from scrapy.core.scheduler import BaseScheduler from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler -from .test_spider_yield_seeds import twisted_sleep +from .test_spider_start import twisted_sleep class MainTestCase(TestCase): @@ -93,7 +93,7 @@ class MainTestCase(TestCase): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) yield Request("data:,a") await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 19f260c0b..d658d1526 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -69,7 +69,7 @@ class AsyncDefNotAsyncioPipeline: class ItemSpider(Spider): name = "itemspider" - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index c2c1b6288..21b5675fd 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_test_yield_seeds(self, test_yield_seeds, spider): - for request in test_yield_seeds: + def process_test_start(self, test_start, spider): + for request in test_start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_test_yield_seeds"] = True + request.cb_kwargs["from_process_test_start"] = True yield request def process_spider_input(self, response, spider): @@ -62,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] - async def yield_seeds(self): + async def start(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -139,10 +139,10 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 2) def parse_spider_mw( - self, response, from_process_spider_input, from_process_test_yield_seeds + self, response, from_process_spider_input, from_process_test_start ): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_test_yield_seeds)) + self.checks.append(bool(from_process_test_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_signals.py b/tests/test_signals.py index 576843f53..de6cf459a 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -10,7 +10,7 @@ from tests.mockserver import MockServer class ItemSpider(Spider): name = "itemspider" - async def yield_seeds(self): + async def start(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} diff --git a/tests/test_spider_yield_seeds.py b/tests/test_spider_start.py similarity index 87% rename from tests/test_spider_yield_seeds.py rename to tests/test_spider_start.py index 8f2a11afb..34f1d5e66 100644 --- a/tests/test_spider_yield_seeds.py +++ b/tests/test_spider_start.py @@ -55,19 +55,19 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f - async def test_yield_seeds(self): + async def test_start(self): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_A await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f - async def test_yield_seeds_subclass(self): + async def test_start_subclass(self): class BaseSpider(Spider): - async def yield_seeds(self): + async def start(self): yield ITEM_A class TestSpider(BaseSpider): @@ -104,7 +104,7 @@ class MainTestCase(TestCase): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_A def start_requests(self): @@ -115,7 +115,7 @@ class MainTestCase(TestCase): @deferred_f_from_coro_f async def test_universal_subclass(self): class BaseSpider(Spider): - async def yield_seeds(self): + async def start(self): yield ITEM_A def start_requests(self): @@ -126,28 +126,28 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) - async def _test_yield_seeds(self, yield_seeds_, expected_items=None): + async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): name = "test" - yield_seeds = yield_seeds_ + start = start_ await self._test_spider(TestSpider, expected_items) @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_asyncio_delayed(self): - async def yield_seeds(spider): + async def start(spider): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) yield ITEM_A - await self._test_yield_seeds(yield_seeds, [ITEM_A]) + await self._test_start(start, [ITEM_A]) @deferred_f_from_coro_f async def test_twisted_delayed(self): - async def yield_seeds(spider): + async def start(spider): await maybe_deferred_to_future( twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) ) yield ITEM_A - await self._test_yield_seeds(yield_seeds, [ITEM_A]) + await self._test_start(start, [ITEM_A]) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9a699e8f6..74744548d 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_test_yield_seeds. + Should work for process_spider_output and, when it's supported, process_test_start. """ ITEM_TYPE: type | tuple @@ -321,12 +321,12 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): class ProcessYieldSeedsSimpleMiddleware: - def process_test_yield_seeds(self, test_yield_seeds, spider): - yield from test_yield_seeds + def process_test_start(self, test_start, spider): + yield from test_start class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): - """process_seeds tests for simple yield_seeds""" + """process_start tests for simple start""" ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware @@ -336,7 +336,7 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): for i in range(2): yield Request(f"https://example.com/{i}", dont_filter=True) yield {"name": "test item"} @@ -347,7 +347,7 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): ) self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - results = yield self.mwman.process_seeds(self.spider) + results = yield self.mwman.process_start(self.spider) return results @inlineCallbacks diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 5e3b2a014..fd2fc3581 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -30,7 +30,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - async def yield_seeds(self): + async def start(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index f157b8c4f..20efac543 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -36,7 +36,7 @@ class RecoverySpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -73,7 +73,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - async def yield_seeds(self): + async def start(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -83,7 +83,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - async def yield_seeds(self): + async def start(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -103,7 +103,7 @@ class GeneratorCallbackSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -140,7 +140,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -215,7 +215,7 @@ class GeneratorOutputChainSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -287,7 +287,7 @@ class NotGeneratorOutputChainSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_spidermiddleware_process_seeds.py b/tests/test_spidermiddleware_process_start.py similarity index 93% rename from tests/test_spidermiddleware_process_seeds.py rename to tests/test_spidermiddleware_process_start.py index 57f28f1fb..7d5cb7cb6 100644 --- a/tests/test_spidermiddleware_process_seeds.py +++ b/tests/test_spidermiddleware_process_start.py @@ -8,7 +8,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from .test_spider_yield_seeds import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep +from .test_spider_start import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} @@ -17,27 +17,27 @@ ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) async for seed in seeds: yield seed class NoOpSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): async for seed in seeds: yield seed class TwistedSleepSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)) async for seed in seeds: yield seed class UniversalSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): async for seed in seeds: yield seed @@ -51,14 +51,14 @@ class UniversalSpiderMiddleware: class ModernWrapSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_B class UniversalWrapSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_B def start_requests(self): @@ -73,7 +73,7 @@ class DeprecatedWrapSpider(Spider): class ModernWrapSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): yield ITEM_A async for seed in seeds: yield seed @@ -81,7 +81,7 @@ class ModernWrapSpiderMiddleware: class UniversalWrapSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): yield ITEM_A async for seed in seeds: yield seed @@ -189,7 +189,7 @@ class MainTestCase(TestCase): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_A await self._test(spider_middlewares, TestSpider, [ITEM_A]) From debab6cff6fb147e7ce8f70c2e493c9568193397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 19 Mar 2025 20:16:49 +0100 Subject: [PATCH 02/45] Support defining Spider.start() as a sync generator --- scrapy/core/spidermw.py | 32 ++++++++++++++++++++++---------- tests/test_spider_start.py | 10 ++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 9a292cae9..95ce47394 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -8,7 +8,12 @@ from __future__ import annotations import logging from collections.abc import AsyncIterable, Callable, Iterable -from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction +from functools import wraps +from inspect import ( + isasyncgenfunction, + iscoroutine, + isgeneratorfunction, +) from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from warnings import warn @@ -49,6 +54,21 @@ def _isiterable(o: Any) -> bool: return isinstance(o, (Iterable, AsyncIterable)) +def _sync_generator_to_async(f: Callable) -> Callable: + @wraps(f) + async def wrapper(*args, **kwargs): + for item in f(*args, **kwargs): + yield item + + return wrapper + + +def _maybe_sync_generator_to_async(f: Callable) -> Callable: + if isgeneratorfunction(f): + return _sync_generator_to_async(f) + return f + + class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" @@ -380,7 +400,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) seeds = as_async_generator(sync_seeds) else: - seeds = yield self._iter_seeds(spider) + seeds = yield _maybe_sync_generator_to_async(spider.start)() seeds = yield self._process_chain("process_start", seeds) return seeds @@ -454,14 +474,6 @@ class SpiderMiddlewareManager(MiddlewareManager): f"https://docs.scrapy.org/en/VERSION/news.html" ) - @staticmethod - def _iter_seeds(spider: Spider): - fn = spider.start - if isasyncgenfunction(fn): - return fn().__aiter__() - assert iscoroutinefunction(fn) - return deferred_from_coro(fn()) - # This method is only needed until _async compatibility methods are removed. @staticmethod def _get_async_method_pair( diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 34f1d5e66..5daa7d209 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -75,6 +75,16 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) + @deferred_f_from_coro_f + async def test_start_sync(self): + class TestSpider(Spider): + name = "test" + + def start(self): + yield ITEM_A + + await self._test_spider(TestSpider, [ITEM_A]) + @deferred_f_from_coro_f async def test_deprecated(self): class TestSpider(Spider): From dfc7cd58e7c5002558313bc3b4b3708e2bf5520e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 19 Mar 2025 20:19:14 +0100 Subject: [PATCH 03/45] Fix issue reported by mypy --- scrapy/commands/check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 59c5ab9b5..8a3cf5078 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -107,10 +107,10 @@ class Command(ScrapyCommand): for method in sorted(methods): print(f" * {method}") else: - start = time.time() + start_time = time.time() self.crawler_process.start() stop = time.time() result.printErrors() - result.printSummary(start, stop) + result.printSummary(start_time, stop) self.exitcode = int(not result.wasSuccessful()) From f91723628afab238145b26af4a2c511bb9f241d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 19 Mar 2025 20:27:38 +0100 Subject: [PATCH 04/45] Disambiguate method refenrece in the docs --- docs/topics/spiders.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a05a4b14d..aae3bb9ba 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -261,8 +261,9 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.start` directly; -to give data more structure you can use :class:`~scrapy.Item` objects: +Instead of :attr:`~.start_urls` you can use :meth:`~scrapy.Spider.start` +directly; to give data more structure you can use :class:`~scrapy.Item` +objects: .. skip: next .. code-block:: python From 07b6fafdd0956a75189e5f4db8602f3f97c252ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 19 Mar 2025 20:53:02 +0100 Subject: [PATCH 05/45] =?UTF-8?q?Rename=20the=20recommended=20name=20for?= =?UTF-8?q?=20the=20process=5Fstart=201st=20parameter=20(seeds=20=E2=86=92?= =?UTF-8?q?=20start)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/spider-middleware.rst | 14 +++--- scrapy/core/engine.py | 16 +++---- scrapy/core/spidermw.py | 14 +++--- .../project/module/middlewares.py.tmpl | 10 ++--- tests/spiders.py | 8 ++-- tests/test_crawl.py | 22 +++++----- tests/test_engine.py | 4 +- tests/test_engine_seeding.py | 12 ++--- tests/test_spider.py | 2 +- tests/test_spidermiddleware.py | 18 ++++---- tests/test_spidermiddleware_process_start.py | 44 +++++++++---------- 11 files changed, 82 insertions(+), 82 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 4632644e6..dc36ff6b6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,7 +70,7 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_start(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] + .. method:: process_start(start: AsyncIterable[Any], /) -> AsyncIterable[Any] :async: Iterate over the output of :meth:`~scrapy.Spider.start` or that @@ -79,12 +79,12 @@ one or more of these methods: .. code-block:: python - async def process_start(self, seeds): - async for seed in seeds: - yield seed + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request You may yield :class:`~scrapy.Request` or :ref:`item <topics-items>` - objects, same as :meth:`~scrapy.Spider.start`, from *seeds* or + objects, same as :meth:`~scrapy.Spider.start`, from *start* or not. To write spider middlewares that work on Scrapy versions lower than @@ -93,8 +93,8 @@ one or more of these methods: .. code-block:: python - def process_start_requests(self, seeds, spider): - yield from seeds + def process_start_requests(self, start, spider): + yield from start .. method:: process_spider_input(response, spider) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 23cddf481..6bf08a890 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -103,7 +103,7 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._seeds: AsyncIterable[Any] | None = None + self._start: AsyncIterable[Any] | None = None self._waiting_for_seed: bool = False def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: @@ -177,13 +177,13 @@ class ExecutionEngine: return self._waiting_for_seed = True try: - seed = yield deferred_from_coro(self._seeds.__anext__()) + seed = yield deferred_from_coro(self._start.__anext__()) except StopAsyncIteration: - self._seeds = None + self._start = None except Exception: - self._seeds = None + self._start = None logger.error( - "Error while reading seeds", + "Error while reading start items and requests", exc_info=True, extra={"spider": self.spider}, ) @@ -201,7 +201,7 @@ class ExecutionEngine: if self._slot is None or self._slot.closing is not None or self.paused: return self._start_scheduled_requests() - if self._seeds is not None and not self._needs_backout(): + if self._start is not None and not self._needs_backout(): yield self._process_next_seed() if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() @@ -289,7 +289,7 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False - if self._seeds is not None: # not all start requests are handled + if self._start is not None: # not all start requests are handled return False return not self._slot.scheduler.has_pending_requests() @@ -380,7 +380,7 @@ class ExecutionEngine: logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._start_next_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - self._seeds = yield self.scraper.spidermw.process_start(spider) + self._start = yield self.scraper.spidermw.process_start(spider) self._slot = _Slot(close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open") and (d := scheduler.open(spider)): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 95ce47394..d5ad4bc8b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -394,15 +394,15 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: - sync_seeds = iter(spider.start_requests()) - sync_seeds = yield self._process_chain( - "process_start_requests", sync_seeds, spider + sync_start = iter(spider.start_requests()) + sync_start = yield self._process_chain( + "process_start_requests", sync_start, spider ) - seeds = as_async_generator(sync_seeds) + start = as_async_generator(sync_start) else: - seeds = yield _maybe_sync_generator_to_async(spider.start)() - seeds = yield self._process_chain("process_start", seeds) - return seeds + start = yield _maybe_sync_generator_to_async(spider.start)() + start = yield self._process_chain("process_start", start) + return start def _check_deprecated_start_requests_use(self, spider: Spider): start_requests_cls = None diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 59a3dca21..3f0239832 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,11 +43,11 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - async def process_start(self, seeds): - # Called with the seeds from the spider start() method or with - # the output of the maching method of an earlier spider middleware. - async for seed in seeds: - yield seed + async def process_start(self, start): + # Called with an async iterator over the spider start() method or the + # maching method of an earlier spider middleware. + async for item_or_request in start: + yield item_or_request def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/tests/spiders.py b/tests/spiders.py index d7fffd001..c47f2bd2b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenYieldSeedsSpider(FollowAllSpider): +class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -345,12 +345,12 @@ class BrokenYieldSeedsSpider(FollowAllSpider): yield from super().parse(response) -class YieldSeedsItemSpider(FollowAllSpider): +class StartItemSpider(FollowAllSpider): async def start(self): yield {"name": "test item"} -class YieldSeedsGoodAndBadOutput(FollowAllSpider): +class StartGoodAndBadOutput(FollowAllSpider): async def start(self): yield {"a": "a"} yield Request("data:,a") @@ -384,7 +384,7 @@ class SingleRequestSpider(MetaSpider): return None -class DuplicateYieldSeedsSpider(MockServerSpider): +class DuplicateStartSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 85d016e44..b5876e3cc 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -34,7 +34,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenYieldSeedsSpider, + BrokenStartSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -43,14 +43,14 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateYieldSeedsSpider, + DuplicateStartSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, - YieldSeedsGoodAndBadOutput, - YieldSeedsItemSpider, + StartGoodAndBadOutput, + StartItemSpider, ) @@ -165,7 +165,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenYieldSeedsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -176,7 +176,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenYieldSeedsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -187,7 +187,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_items(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(YieldSeedsItemSpider) + crawler = get_crawler(StartItemSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 @@ -199,7 +199,7 @@ class TestCrawl(TestCase): things fail when ItemAdapter is actually used on the corresponding non-item object.""" with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(YieldSeedsGoodAndBadOutput) + crawler = get_crawler(StartGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 @@ -207,7 +207,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenYieldSeedsSpider, settings) + crawler = get_crawler(BrokenStartSpider, settings) yield crawler.crawl(mockserver=self.mockserver) assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( 99 @@ -216,13 +216,13 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateYieldSeedsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) assert crawler.spider.visited == 6 - crawler = get_crawler(DuplicateYieldSeedsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, diff --git a/tests/test_engine.py b/tests/test_engine.py index 306a0e775..4dc77e438 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -494,11 +494,11 @@ def test_request_scheduled_signal(caplog): engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() - async def seeds(): + async def start(): return yield - engine._seeds = seeds() + engine._start = start() engine._slot = _Slot(False, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index 03a5ac64a..ad0dc9c12 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -15,9 +15,9 @@ from .test_spider_start import twisted_sleep class MainTestCase(TestCase): @inlineCallbacks - def test_scheduler_priority_over_seeds_simple(self): - """Scrapy reads seeds into the scheduler while the scheduler is empty, - but otherwise prioritizes requests already in the scheduler. + def test_scheduler_priority_over_start_simple(self): + """Scrapy reads start() into the scheduler while the scheduler is + empty, but otherwise prioritizes requests already in the scheduler. This test shows how, given a scheduler pre-filled with a request, that request is sent before sending the first seed request. @@ -61,13 +61,13 @@ class MainTestCase(TestCase): assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" @inlineCallbacks - def test_scheduler_priority_over_seeds_complex(self): - """Although Scrapy reads seeds into the scheduler while the scheduler + def test_scheduler_priority_over_start_complex(self): + """Although Scrapy reads start() into the scheduler while the scheduler is empty and otherwise prioritizes requests already in the scheduler, this is done in a non-blocking way. That is, if the scheduler reports having requests but yields none, - requests from seeds will be scheduled. + requests from start() will be scheduled. """ class TestScheduler(BaseScheduler): diff --git a/tests/test_spider.py b/tests/test_spider.py index b7ddb37c4..c3d66617d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -460,7 +460,7 @@ class TestCrawlSpider(TestSpider): ( "scrapy.core.engine", "ERROR", - "Error while reading seeds", + "Error while reading start items and requests", ), ) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 74744548d..3d8c885d1 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -320,19 +320,19 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): ) -class ProcessYieldSeedsSimpleMiddleware: +class ProcessStartSimpleMiddleware: def process_test_start(self, test_start, spider): yield from test_start -class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): +class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): """process_start tests for simple start""" ITEM_TYPE = (Request, dict) - MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware + MW_SIMPLE = ProcessStartSimpleMiddleware @inlineCallbacks - def _get_processed_seeds(self, *mw_classes): + def _get_processed_start(self, *mw_classes): class TestSpider(Spider): name = "test" @@ -353,11 +353,11 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): @inlineCallbacks def test_simple(self): """Simple mw""" - seeds = yield self._get_processed_seeds(self.MW_SIMPLE) - assert isasyncgen(seeds) - seed_list = yield deferred_from_coro(collect_asyncgen(seeds)) - assert len(seed_list) == self.RESULT_COUNT - assert isinstance(seed_list[0], self.ITEM_TYPE) + start = yield self._get_processed_start(self.MW_SIMPLE) + assert isasyncgen(start) + start_list = yield deferred_from_coro(collect_asyncgen(start)) + assert len(start_list) == self.RESULT_COUNT + assert isinstance(start_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 7d5cb7cb6..6dced5126 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -17,29 +17,29 @@ ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request class NoOpSpiderMiddleware: - async def process_start(self, seeds): - async for seed in seeds: - yield seed + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request class TwistedSleepSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)) - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request class UniversalSpiderMiddleware: - async def process_start(self, seeds): - async for seed in seeds: - yield seed + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request def process_start_requests(self, start_requests, spider): raise NotImplementedError @@ -73,30 +73,30 @@ class DeprecatedWrapSpider(Spider): class ModernWrapSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): yield ITEM_A - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request yield ITEM_C class UniversalWrapSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): yield ITEM_A - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request yield ITEM_C - def process_start_requests(self, seeds, spider): + def process_start_requests(self, start, spider): yield ITEM_A - yield from seeds + yield from start yield ITEM_C class DeprecatedWrapSpiderMiddleware: - def process_start_requests(self, seeds, spider): + def process_start_requests(self, start, spider): yield ITEM_A - yield from seeds + yield from start yield ITEM_C From 28b78457aa7287859092e6013439242f50e4f61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 19 Mar 2025 21:33:33 +0100 Subject: [PATCH 06/45] Update test expectations --- tests/test_commands.py | 3 ++- tests/test_crawl.py | 21 --------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7d399d474..16af97842 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -683,6 +683,7 @@ class BadSpider(scrapy.Spider): name = "bad" async def start(self): raise Exception("oops!") + yield """ @contextmanager @@ -775,7 +776,7 @@ class MySpider(scrapy.Spider): def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.py") assert "start" in log - assert "badspider.py" in log + assert "badspider.py" in log, log def test_asyncio_enabled_true(self): log = self.get_log( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b5876e3cc..919c599f7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -360,27 +360,6 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks - def test_graceful_crawl_error_handling(self): - """ - Test whether errors happening anywhere in Crawler.crawl() are properly - reported (and not somehow swallowed) after a graceful engine shutdown. - The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.test_start(), - SpiderMiddleware.process_test_start(), etc. - """ - - class TestError(Exception): - pass - - class FaultySpider(SimpleSpider): - async def start(self): - raise TestError - - crawler = get_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - assert not crawler.crawling - @defer.inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { From 2ee01efe496db9ff8506ca37fe2571f5f4ac2849 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 20 Mar 2025 07:49:44 -0300 Subject: [PATCH 07/45] feat: Add count to spider_exceptions stats (#6740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add overall exception_count to spider_exceptions stats * Remove variable * Update test_closespider.py * Update test_closespider.py * Rename exception_count → count --- scrapy/core/scraper.py | 1 + tests/test_closespider.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index b664b61f6..496adb500 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -246,6 +246,7 @@ class Scraper: spider=spider, ) assert self.crawler.stats + self.crawler.stats.inc_value("spider_exceptions/count", spider=spider) self.crawler.stats.inc_value( f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 476662789..4a17b254b 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -88,6 +88,7 @@ class TestCloseSpider(TestCase): assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) + assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on @defer.inlineCallbacks From 0867ecc0c74b6922408787bf940b255a125c58fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 20 Mar 2025 12:09:25 +0100 Subject: [PATCH 08/45] Revert "Support defining Spider.start() as a sync generator" This reverts commit debab6cff6fb147e7ce8f70c2e493c9568193397. --- scrapy/core/spidermw.py | 32 ++++++++++---------------------- tests/test_spider_start.py | 10 ---------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d5ad4bc8b..7b3826d21 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -8,12 +8,7 @@ from __future__ import annotations import logging from collections.abc import AsyncIterable, Callable, Iterable -from functools import wraps -from inspect import ( - isasyncgenfunction, - iscoroutine, - isgeneratorfunction, -) +from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from warnings import warn @@ -54,21 +49,6 @@ def _isiterable(o: Any) -> bool: return isinstance(o, (Iterable, AsyncIterable)) -def _sync_generator_to_async(f: Callable) -> Callable: - @wraps(f) - async def wrapper(*args, **kwargs): - for item in f(*args, **kwargs): - yield item - - return wrapper - - -def _maybe_sync_generator_to_async(f: Callable) -> Callable: - if isgeneratorfunction(f): - return _sync_generator_to_async(f) - return f - - class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" @@ -400,7 +380,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) start = as_async_generator(sync_start) else: - start = yield _maybe_sync_generator_to_async(spider.start)() + start = yield self._iter_seeds(spider) start = yield self._process_chain("process_start", start) return start @@ -474,6 +454,14 @@ class SpiderMiddlewareManager(MiddlewareManager): f"https://docs.scrapy.org/en/VERSION/news.html" ) + @staticmethod + def _iter_seeds(spider: Spider): + fn = spider.start + if isasyncgenfunction(fn): + return fn().__aiter__() + assert iscoroutinefunction(fn) + return deferred_from_coro(fn()) + # This method is only needed until _async compatibility methods are removed. @staticmethod def _get_async_method_pair( diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 5daa7d209..34f1d5e66 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -75,16 +75,6 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f - async def test_start_sync(self): - class TestSpider(Spider): - name = "test" - - def start(self): - yield ITEM_A - - await self._test_spider(TestSpider, [ITEM_A]) - @deferred_f_from_coro_f async def test_deprecated(self): class TestSpider(Spider): From 3ca882fba86750199c2f41ef24b5495a4afa7988 Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Thu, 20 Mar 2025 18:02:10 +0600 Subject: [PATCH 09/45] Syntax Error Fixed (#6738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Syntax error fix issue #6731 * test case added * extra logic removed * mock spider fixture * Update scrapy/utils/misc.py Co-authored-by: Adrián Chaves <adrian@chaves.gal> * settings.rst updated * settings.rst updated * settings.rst updated --------- Co-authored-by: Adrián Chaves <adrian@chaves.gal> --- docs/topics/settings.rst | 15 +++ scrapy/settings/default_settings.py | 2 + scrapy/utils/misc.py | 2 + ...t_return_with_argument_inside_generator.py | 110 +++++++++++++----- 4 files changed, 98 insertions(+), 31 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ca0af569f..a59a61050 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2047,6 +2047,21 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and there is no overriding User-Agent header specified for the request. +.. setting:: WARN_ON_GENERATOR_RETURN_VALUE + +WARN_ON_GENERATOR_RETURN_VALUE +------------------------------ + +Default: ``True`` + +When enabled, Scrapy will warn if generator-based callback methods (like +``parse``) contain return statements with non-``None`` values. This helps detect +potential mistakes in spider development. + +Disable this setting to prevent syntax errors that may occur when dynamically +modifying generator function source code during runtime, skip AST parsing of +callback functions, or improve performance in auto-reloading development +environments. Settings documented elsewhere: ------------------------------ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 645e50301..680fded7a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -351,3 +351,5 @@ SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } + +WARN_ON_GENERATOR_RETURN_VALUE = True diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d319e7950..b7b436260 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -286,6 +286,8 @@ def warn_on_generator_with_return_value( Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ + if not spider.settings.getbool("WARN_ON_GENERATOR_RETURN_VALUE"): + return try: if is_generator_with_return_value(callable): warnings.warn( diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 81a83c3d7..ad31e5185 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -2,6 +2,8 @@ import warnings from functools import partial from unittest import mock +import pytest + from scrapy.utils.misc import ( is_generator_with_return_value, warn_on_generator_with_return_value, @@ -40,7 +42,24 @@ def generator_that_returns_stuff(): class TestUtilsMisc: - def test_generators_return_something(self): + @pytest.fixture + def mock_spider(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings() + + return MockSpider() + + def test_generators_return_something(self, mock_spider): def f1(): yield 1 return 2 @@ -75,30 +94,30 @@ https://example.org assert is_generator_with_return_value(i1) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_something) + warn_on_generator_with_return_value(mock_spider, top_level_return_something) assert len(w) == 1 assert ( - 'The "NoneType.top_level_return_something" method is a generator' + 'The "MockSpider.top_level_return_something" method is a generator' in str(w[0].message) ) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f1) + warn_on_generator_with_return_value(mock_spider, f1) assert len(w) == 1 - assert 'The "NoneType.f1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g1) + warn_on_generator_with_return_value(mock_spider, g1) assert len(w) == 1 - assert 'The "NoneType.g1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h1) + warn_on_generator_with_return_value(mock_spider, h1) assert len(w) == 1 - assert 'The "NoneType.h1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i1) + warn_on_generator_with_return_value(mock_spider, i1) assert len(w) == 1 - assert 'The "NoneType.i1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) - def test_generators_return_none(self): + def test_generators_return_none(self, mock_spider): def f2(): yield 1 @@ -142,31 +161,31 @@ https://example.org assert not is_generator_with_return_value(l2) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f2) + warn_on_generator_with_return_value(mock_spider, f2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g2) + warn_on_generator_with_return_value(mock_spider, g2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h2) + warn_on_generator_with_return_value(mock_spider, h2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i2) + warn_on_generator_with_return_value(mock_spider, i2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j2) + warn_on_generator_with_return_value(mock_spider, j2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k2) + warn_on_generator_with_return_value(mock_spider, k2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l2) + warn_on_generator_with_return_value(mock_spider, l2) assert len(w) == 0 - def test_generators_return_none_with_decorator(self): + def test_generators_return_none_with_decorator(self, mock_spider): def decorator(func): def inner_func(): func() @@ -223,36 +242,36 @@ https://example.org assert not is_generator_with_return_value(l3) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f3) + warn_on_generator_with_return_value(mock_spider, f3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g3) + warn_on_generator_with_return_value(mock_spider, g3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h3) + warn_on_generator_with_return_value(mock_spider, h3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i3) + warn_on_generator_with_return_value(mock_spider, i3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j3) + warn_on_generator_with_return_value(mock_spider, j3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k3) + warn_on_generator_with_return_value(mock_spider, k3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l3) + warn_on_generator_with_return_value(mock_spider, l3) assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) - def test_indentation_error(self): + def test_indentation_error(self, mock_spider): with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 1 assert "Unable to determine" in str(w[0].message) @@ -262,3 +281,32 @@ https://example.org partial_cb = partial(cb, arg1=42) assert not is_generator_with_return_value(partial_cb) + + def test_warn_on_generator_with_return_value_settings_disabled(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 0 + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 1 + assert "is a generator" in str(w[0].message) From e50914e0f5b98ee4c9cb1f182ec2cd684fdf9900 Mon Sep 17 00:00:00 2001 From: Suejung Shin <suejung.shin@sentry.io> Date: Fri, 21 Mar 2025 04:28:47 -0700 Subject: [PATCH 10/45] Codecov: Add test analytics (#6741) --- .github/workflows/tests-macos.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ .github/workflows/tests-windows.yml | 4 ++++ .gitignore | 1 + tox.ini | 8 ++++---- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index ce0e1a6c2..d740808cc 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -33,3 +33,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 444aa3557..34819f227 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -88,3 +88,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 537a01e29..bbbb704e5 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -64,3 +64,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.gitignore b/.gitignore index 6c5c50e08..0a3f0ac1c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ htmlcov/ .pytest_cache/ .coverage.* coverage.* +*.junit.xml test-output.* .cache/ .mypy_cache/ diff --git a/tox.ini b/tox.ini index eb084f0f5..59572442d 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --durations=10 docs scrapy tests --doctest-modules} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 docs scrapy tests --doctest-modules} install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -118,7 +118,7 @@ install_command = python -I -m pip install {opts} {packages} commands = ; tests for docs fail with parsel < 1.8.0 - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -254,7 +254,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -265,4 +265,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy -m requires_botocore} From 9f99da8f865efff11f6c8736567b8fbd4413091c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 24 Mar 2025 13:26:25 +0500 Subject: [PATCH 11/45] Convert test_downloadermiddleware_robotstxt.py from callbacks to awaits. (#6743) --- tests/test_downloadermiddleware_robotstxt.py | 168 +++++++++---------- 1 file changed, 76 insertions(+), 92 deletions(-) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ad335f852..9518f1835 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,9 +1,11 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING from unittest import mock import pytest from twisted.internet import error, reactor -from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred from twisted.python import failure from twisted.trial import unittest @@ -13,8 +15,12 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from tests.test_robotstxt_interface import rerp_available +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class TestRobotsTxtMiddleware(unittest.TestCase): def setUp(self): @@ -31,7 +37,7 @@ class TestRobotsTxtMiddleware(unittest.TestCase): with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self): + def _get_successful_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -54,54 +60,41 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt(self): + @deferred_f_from_coro_f + async def test_robotstxt(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"), - self.assertIgnored(Request("http://site.local/admin/main"), middleware), - self.assertIgnored(Request("http://site.local/static/"), middleware), - self.assertIgnored( - Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), - middleware, - ), - self.assertIgnored( - Request("http://site.local/wiki/Käyttäjä:"), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + self.assertRobotsTxtRequested("http://site.local") + await self.assertIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertIgnored(Request("http://site.local/static/"), middleware) + await self.assertIgnored( + Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware + ) + await self.assertIgnored( + Request("http://site.local/wiki/Käyttäjä:"), middleware ) - def test_robotstxt_ready_parser(self): + @deferred_f_from_coro_f + async def test_robotstxt_ready_parser(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback( - lambda _: self.assertNotIgnored( - Request("http://site.local/allowed"), middleware - ) - ) - return d + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - def test_robotstxt_meta(self): + @deferred_f_from_coro_f + async def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) meta = {"dont_obey_robotstxt": True} - return DeferredList( - [ - self.assertNotIgnored( - Request("http://site.local/allowed", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/admin/main", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/static/", meta=meta), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored( + Request("http://site.local/allowed", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/admin/main", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self): + def _get_garbage_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( @@ -116,22 +109,16 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt_garbage(self): + @deferred_f_from_coro_f + async def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local"), middleware), - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self): + def _get_emptybody_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") @@ -144,21 +131,16 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt_empty_response(self): + @deferred_f_from_coro_f + async def test_robotstxt_empty_response(self): # empty response should equal 'allow all' middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def test_robotstxt_error(self): + @deferred_f_from_coro_f + async def test_robotstxt_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -171,15 +153,13 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) - deferred = middleware.process_request(Request("http://site.local"), None) + await maybe_deferred_to_future( + middleware.process_request(Request("http://site.local"), None) + ) + assert middleware._logerror.called - def check_called(_: Any) -> None: - assert middleware._logerror.called - - deferred.addCallback(check_called) - return deferred - - def test_robotstxt_immediate_error(self): + @deferred_f_from_coro_f + async def test_robotstxt_immediate_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -191,9 +171,10 @@ Disallow: /some/randome/page.html self.crawler.engine.download.side_effect = immediate_failure middleware = RobotsTxtMiddleware(self.crawler) - return self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local"), middleware) - def test_ignore_robotstxt_request(self): + @deferred_f_from_coro_f + async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): @@ -206,13 +187,8 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) mw_module_logger.error = mock.MagicMock() - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - - def check_not_called(_: Any) -> None: - assert not mw_module_logger.error.called # type: ignore[attr-defined] - - d.addCallback(check_not_called) - return d + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + assert not mw_module_logger.error.called # type: ignore[attr-defined] def test_robotstxt_user_agent_setting(self): crawler = self._get_successful_crawler() @@ -236,19 +212,27 @@ Disallow: /some/randome/page.html Deferred, ) - def assertNotIgnored(self, request, middleware): + async def assertNotIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: spider = None # not actually used - dfd = maybeDeferred(middleware.process_request, request, spider) - dfd.addCallback(self.assertIsNone) - return dfd + result = await maybe_deferred_to_future( + maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload] + ) + assert result is None - def assertIgnored(self, request, middleware): + async def assertIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: spider = None # not actually used - return self.assertFailure( - maybeDeferred(middleware.process_request, request, spider), IgnoreRequest + await maybe_deferred_to_future( + self.assertFailure( + middleware.process_request(request, spider), # type: ignore[arg-type] + IgnoreRequest, + ) ) - def assertRobotsTxtRequested(self, base_url): + def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download.call_args_list request = calls[0][0][0] assert request.url == f"{base_url}/robots.txt" From cc49ea9f18889fa658734f79a2a9523e84a50874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 24 Mar 2025 10:58:19 +0100 Subject: [PATCH 12/45] =?UTF-8?q?Inclued=20()=20in=20=E2=80=A6start=5Frequ?= =?UTF-8?q?ests=20mentions=20on=20news.rst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 18 +++++++++--------- sep/sep-018.rst | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e210ebdb1..b88626bd5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,7 +10,7 @@ Scrapy VERSION (unreleased) Highlights: -- Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.start` +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` (async) and changed how it is iterated by default. Backward-incompatible changes @@ -24,8 +24,8 @@ Backward-incompatible changes - In ``scrapy.core.engine.ExecutionEngine``: - - The second parameter of ``open_spider()``, ``start_requests``, has been - removed. The starting requests are determined by the ``spider`` + - The second parameter of ``open_spider()``, ``start_requests()``, has + been removed. The starting requests are determined by the ``spider`` parameter instead (see :meth:`~scrapy.Spider.start`). - The ``slot`` attribute has been renamed to ``_slot`` and should not be @@ -104,7 +104,7 @@ Highlights: - Dropped support for Python 3.8, added support for Python 3.13 -- ``scrapy.Spider.start_requests`` can now yield items +- ``scrapy.Spider.start_requests()`` can now yield items - Added :class:`~scrapy.http.JsonResponse` @@ -891,7 +891,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - ``scrapy.Spider.start_requests`` or in a handler of the + ``scrapy.Spider.start_requests()`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now @@ -3467,7 +3467,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement ``scrapy.spiders.Spider.start_requests``, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -6388,7 +6388,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix ``start_requests`` laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -6581,7 +6581,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added a ``process_start_requests`` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -6644,7 +6644,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch ``start_requests`` iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e6d601fe1..29b1f860e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -619,7 +619,7 @@ Resolved: ``manager.scraper.process_request()`` instead of ``manager.engine.crawl()`` - should we support adding additional start requests from a spider middleware? - - Yes - there is a spider middleware method (``start_requests``) for that + - Yes - there is a spider middleware method (``start_requests()``) for that - should ``process_response()`` receive a ``request`` argument with the ``request`` that originated it?. ``response.request`` is the latest request, not the original one (think of redirections), but it does carry the ``meta`` From b37f3b5422fbca2b8949b1e48dace50aeadc3f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 24 Mar 2025 11:22:41 +0100 Subject: [PATCH 13/45] Fix text replacement artifacts --- tests/test_crawl.py | 4 ++-- tests/test_request_cb_kwargs.py | 12 +++++------- tests/test_spidermiddleware.py | 7 ++++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 92b6238d7..9115edd27 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -305,10 +305,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # test_start doesn't set Referer header + # start() doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to test_start url + # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 21b5675fd..79b53b33b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_test_start(self, test_start, spider): - for request in test_start: + async def process_start(self, start): + async for request in start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_test_start"] = True + request.cb_kwargs["from_process_start"] = True yield request def process_spider_input(self, response, spider): @@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(bool(from_process_response)) self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw( - self, response, from_process_spider_input, from_process_test_start - ): + def parse_spider_mw(self, response, from_process_spider_input, from_process_start): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_test_start)) + self.checks.append(bool(from_process_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 3d8c885d1..9728d0d81 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_test_start. + Should work for process_spider_output and, when it's supported, process_start. """ ITEM_TYPE: type | tuple @@ -321,8 +321,9 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): class ProcessStartSimpleMiddleware: - def process_test_start(self, test_start, spider): - yield from test_start + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): From 4ac921f3891268b96bed711ddc75eff0e56c958f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 24 Mar 2025 17:26:58 +0100 Subject: [PATCH 14/45] WIP --- docs/news.rst | 18 -- docs/topics/request-response.rst | 9 +- docs/topics/settings.rst | 23 -- docs/topics/spider-middleware.rst | 17 -- docs/topics/spiders.rst | 51 ++++ scrapy/__init__.py | 2 - scrapy/commands/shell.py | 2 - scrapy/core/_seeding.py | 68 ------ scrapy/core/engine.py | 93 ++------ scrapy/http/request/__init__.py | 12 +- scrapy/settings/default_settings.py | 4 - scrapy/spiders/__init__.py | 16 +- tests/test_crawl.py | 20 +- tests/test_engine_loop.py | 130 ++++++++++ tests/test_engine_seeding.py | 358 ---------------------------- tests/test_scheduler.py | 35 ++- 16 files changed, 253 insertions(+), 605 deletions(-) delete mode 100644 scrapy/core/_seeding.py create mode 100644 tests/test_engine_loop.py delete mode 100644 tests/test_engine_seeding.py diff --git a/docs/news.rst b/docs/news.rst index b88626bd5..37ed1e693 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,9 +19,6 @@ Backward-incompatible changes - By default, the iteration of start requests and items no longer stops once there are requests in the scheduler. - You can restore the previous behavior by setting :setting:`SEEDING_POLICY` - to :py:enum:mem:`~scrapy.SeedingPolicy.lazy`. - - In ``scrapy.core.engine.ExecutionEngine``: - The second parameter of ``open_spider()``, ``start_requests()``, has @@ -69,21 +66,6 @@ New features (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) -- The new :setting:`SEEDING_POLICY` setting allows customizing how start - requests and items are iterated. - - You can also override the active seeding policy from - :meth:`Spider.start <scrapy.Spider.start>` and from - :meth:`SpiderMiddleware.process_start - <scrapy.spidermiddlewares.SpiderMiddleware.process_start>`. - - .. note:: Some third-party spider middlewares may need to be updated for - Scrapy VERSION support before you can use them in combination with the - ability to override the active seeding policy. - - (:issue:`740`, :issue:`1051`, :issue:`1443`, :issue:`3237`, :issue:`4467`, - :issue:`5282`, :issue:`6729`) - Bug fixes ~~~~~~~~~ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 2032fc5cf..de7840878 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -127,10 +127,7 @@ Request objects body to bytes (if given as a string). :type encoding: str - :param priority: the priority of this request (defaults to ``0``). - The priority is used by the scheduler to define the order used to process - requests. Requests with a higher priority value will execute earlier. - Negative values are allowed in order to indicate relatively low-priority. + :param priority: sets :attr:`priority`, defaults to ``0``. :type priority: int :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. @@ -179,6 +176,8 @@ Request objects .. autoattribute:: errback + .. autoattribute:: priority + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents @@ -353,7 +352,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - async def start(self): + async def yield_seeds(self): for u in self.start_urls: yield scrapy.Request( u, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 261100e00..67de4f5f1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1733,29 +1733,6 @@ Soft limit (in bytes) for response data being processed. While the sum of the sizes of all responses being processed is above this value, Scrapy does not process new requests. -.. setting:: SEEDING_POLICY - -SEEDING_POLICY --------------- - -.. versionadded:: VERSION - -Default: :py:enum:mem:`SeedingPolicy.greedy <scrapy.SeedingPolicy.greedy>` - -Determines the way :meth:`Spider.start <scrapy.Spider.start>` is -iterated. - -Its value may be defined as a member of the :class:`~scrapy.SeedingPolicy` enum -(e.g. :py:enum:mem:`SeedingPolicy.lazy <scrapy.SeedingPolicy.lazy>`) or as a -matching string (e.g. ``"lazy"``). - -You can also override the active seeding policy from :meth:`Spider.start -<scrapy.Spider.start>` and from :meth:`SpiderMiddleware.process_start -<scrapy.spidermiddlewares.SpiderMiddleware.process_start>`. - -.. autoenum:: scrapy.SeedingPolicy - :members: - .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 79b8b1555..0baf094bc 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -85,23 +85,6 @@ one or more of these methods: You may yield the same type of objects as :meth:`~scrapy.Spider.start`. - As with :meth:`~scrapy.Spider.start`, how this method is iterated by - default is controlled by :setting:`SEEDING_POLICY`. It is also possible - to yield a :class:`~scrapy.SeedingPolicy` enum or a matching string to - change the active seeding policy, for example: - - .. code-block:: python - - async def process_start(self, start): - yield "front_load" - async for item_or_request in start: - yield item_or_request - yield "idle" - - .. tip:: You can also restore the configured seeding policy by - :ref:`reading its value <component-settings>` from the - :setting:`SEEDING_POLICY` setting and yielding it. - To write spider middlewares that work on Scrapy versions lower than VERSION, define also a synchronous ``process_start_requests()`` method that returns an iterable. For example: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 71775247c..2432223eb 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -364,6 +364,57 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +.. _spider-start: + +Spider start +============ + +The way :meth:`~scrapy.Spider.start` works may be counterintuitive. + +By default, if you do not use :ref:`await <await>` in +:meth:`~scrapy.Spider.start`, or if you instead use the +:attr:`~scrapy.Spider.start_urls` attribute, this is what happens: + +- The first 8-16 start requests (based on :setting:`CONCURRENT_REQUESTS` and + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) are sent in the order in which + they are yielded. + + .. note:: Responses may come in a different order. + +- The remaining start requests are sent in reverse order, and only when there + are not enough pending requests yielded from callbacks to reach the + configured concurrency. + + This is because the :ref:`scheduler <topics-scheduler>`, where pending + requests are stored, uses a LIFO (last in, first out) queue by default, + configured in the :setting:`SCHEDULER_MEMORY_QUEUE` and + :setting:`SCHEDULER_DISK_QUEUE`. + + So, provided all pending requests have the same + :attr:`~scrapy.Request.priority`, scheduled requests are sent in reserve + order. The first few requests are sent in order only because they are sent + as soon as they are scheduled. + +If you need start requests to be sent before requests yielded from spider +callbacks, you can set a higher priority for them. For example: + +.. code-block:: python + + async def start(self): + async for request in super().start(): + yield request.replace(priority=1) + +If you also need them to be sent in order, you can assign them decreasing +priority values. For example: + +.. code-block:: python + + async def start(self): + priority = len(self.start_urls) + async for request in super().start(): + yield request.replace(priority=priority) + priority -= 1 + .. _builtin-spiders: Generic Spiders diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 7bb958a2d..256504c9c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,7 +7,6 @@ import sys import warnings # Declare top-level shortcuts -from scrapy.core._seeding import SeedingPolicy from scrapy.http import FormRequest, Request from scrapy.item import Field, Item from scrapy.selector import Selector @@ -18,7 +17,6 @@ __all__ = [ "FormRequest", "Item", "Request", - "SeedingPolicy", "Selector", "Spider", "__version__", diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index c50c963ef..4c3aca605 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,7 +9,6 @@ from __future__ import annotations from threading import Thread from typing import TYPE_CHECKING, Any -from scrapy import SeedingPolicy from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -28,7 +27,6 @@ class Command(ScrapyCommand): "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "SEEDING_POLICY": SeedingPolicy.lazy, } def syntax(self) -> str: diff --git a/scrapy/core/_seeding.py b/scrapy/core/_seeding.py deleted file mode 100644 index 08c68951c..000000000 --- a/scrapy/core/_seeding.py +++ /dev/null @@ -1,68 +0,0 @@ -from enum import Enum - -try: - from enum_tools.documentation import document_enum -except ImportError: - - def document_enum(func): # type: ignore[misc] - return func -else: - # https://github.com/domdfcoding/enum_tools/issues/29 - import enum_tools.documentation - - enum_tools.documentation.INTERACTIVE = True - - -@document_enum -class SeedingPolicy(Enum): - front_load = "front_load" - """The crawl does not start until all start requests have been scheduled. - - Aims to give the :ref:`scheduler <topics-scheduler>` full control over - request order from the start. Some custom schedulers may require this - seeding policy to work as designed. - """ - - greedy = "greedy" - """Iterating start items and requests takes priority over processing - scheduled requests. - - Every time a start request is iterated, it is scheduled, and then the next - request from the scheduler is sent. - - .. note:: That request sent may not be the scheduled start request - depending on the priority of scheduled requests, on the configured - :setting:`SCHEDULER` and on certain scheduler settings (e.g. - :setting:`SCHEDULER_MEMORY_QUEUE`). - - Best used when prioritizing start requests is important. - """ - - idle = "idle" - """A single start item or request is read only when there are neither - scheduled nor on-going requests. - - That is, a new start item or request is not read until all requests - triggered by the previous start request, directly or indirectly, have been - processed. - - Unlike :py:enum:mem:`lazy`, resource savings are prioritized over crawl - speed. - - It is functionally equivalent to running a spider multiple times in a row, - one per start request. - """ - - lazy = "lazy" - """Processing scheduled requests takes priority over iterating start items - and requests. - - Aims to minimize the number of requests in the scheduler at any given time, - to minimize resource usage (memory or disk, depending on - :setting:`JOBDIR`). - - It is best used when start request priority is not important. - - Switching to :py:enum:mem:`idle` may lower resource usage further at the - cost of also lowering crawl speed. - """ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b87fd1b78..6ec9780e6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -24,8 +24,6 @@ from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce -from ._seeding import SeedingPolicy - if TYPE_CHECKING: from collections.abc import AsyncIterable, Callable, Generator @@ -43,10 +41,6 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -class _SeedingPolicyChange(Exception): - pass - - class _Slot: def __init__( self, @@ -109,20 +103,7 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._load_seeding_policy() self._start: AsyncIterable[Any] | None = None - self._waiting_for_seed: bool = False - - def _load_seeding_policy(self) -> None: - try: - self._seeding_policy = SeedingPolicy(self.settings["SEEDING_POLICY"]) - except ValueError: - supported_values = ", ".join(policy.value for policy in SeedingPolicy) - raise ValueError( - f"The value of the SEEDING_POLICY setting " - f"({self.settings['SEEDING_POLICY']!r}) is not supported. " - f"Supported values: {supported_values}." - ) def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -185,10 +166,7 @@ class ExecutionEngine: self.paused = False @inlineCallbacks - def _process_next_seed(self): - if self._waiting_for_seed: - return - self._waiting_for_seed = True + def _process_next_spider_start_yield(self): try: item_or_request = yield deferred_from_coro(self._start.__anext__()) except StopAsyncIteration: @@ -203,68 +181,28 @@ class ExecutionEngine: else: if isinstance(item_or_request, Request): self.crawl(item_or_request) - if ( - self._seeding_policy is not SeedingPolicy.front_load - and not self._needs_backout() - ): - self._start_scheduled_request() - elif isinstance(item_or_request, (str, SeedingPolicy)): - try: - self._seeding_policy = SeedingPolicy(item_or_request) - except ValueError: - valid_policy_strings = ", ".join( - policy.value for policy in SeedingPolicy - ) - logger.error( - f"Start value {item_or_request!r} has been ignored. " - f"Start values of {str} type must be valid seeding " - f"policies ({valid_policy_strings})." - ) - self._slot.nextcall.schedule() - else: - raise _SeedingPolicyChange else: self.scraper.start_itemproc(item_or_request, response=None) self._slot.nextcall.schedule() - finally: - self._waiting_for_seed = False - if self._seeding_policy is SeedingPolicy.front_load and self._start is None: - self._slot.nextcall.schedule() @inlineCallbacks - def _start_next_requests(self) -> Generator[Deferred[Any], Any, None]: + def _process_spider_start(self) -> Generator[Deferred[Any], Any, None]: + """Process start items and requests in an asynchronous loop. + + Items are scraped. Requests are scheduled. + """ + while self._start is not None: + yield self._process_next_spider_start_yield() + if not self._needs_backout(): + self._slot.nextcall.schedule() + + def _start_next_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: return - try: - if self._seeding_policy in {SeedingPolicy.idle, SeedingPolicy.lazy}: - while not self._needs_backout(): - if self._start_scheduled_request() is None: - break - if ( - self._start is not None - and not self._needs_backout() - and ( - self._seeding_policy is not SeedingPolicy.idle - or (not self._waiting_for_seed and not self.downloader.active) - ) - ): - yield self._process_next_seed() - else: - assert self._seeding_policy in { - SeedingPolicy.front_load, - SeedingPolicy.greedy, - } - if self._start is not None: - if not self._needs_backout(): - yield self._process_next_seed() - else: - while not self._needs_backout(): - if self._start_scheduled_request() is None: - break - except _SeedingPolicyChange: - self._slot.nextcall.schedule() - return + while not self._needs_backout(): + if self._start_scheduled_request() is None: + break if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() @@ -452,6 +390,7 @@ class ExecutionEngine: assert self.crawler.stats self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) + self._process_spider_start() self._slot.nextcall.schedule() self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index d72c72ecd..0bfc92c6b 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -130,6 +130,16 @@ class Request(object_ref): self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") + + #: Default: ``0`` + #: + #: Value that the :ref:`scheduler <topics-scheduler>` may use for + #: request prioritization. + #: + #: Built-in schedulers prioritize requests with a higher priority + #: value. + #: + #: Negative values are allowed. self.priority: int = priority if not (callable(callback) or callback is None): @@ -191,7 +201,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.start`. + #: default. See :meth:`~scrapy.Spider.yield_seeds`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 886154bfa..645e50301 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,8 +17,6 @@ import sys from importlib import import_module from pathlib import Path -from scrapy import SeedingPolicy - ADDONS = {} AJAXCRAWL_ENABLED = False @@ -310,8 +308,6 @@ SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 -SEEDING_POLICY = SeedingPolicy.greedy - SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 6eba1b188..30e197b8c 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -113,20 +113,6 @@ class Spider(object_ref): async def start(self): yield {"foo": "bar"} - Use :setting:`SEEDING_POLICY` to set how :meth:`start` is - iterated by default. It is also - possible to yield a :class:`~scrapy.SeedingPolicy` enum or a matching - string to change the active seeding policy, for example: - - .. code-block:: python - - async def start(self): - yield "front_load" - yield Request("https://a.example") - yield Request("https://b.example") - yield self.crawler.settings["SEEDING_POLICY"] - yield Request("https://c.example") - To write spiders that work on Scrapy versions lower than VERSION, define also a synchronous ``start_requests()`` method that returns an iterable. For example: @@ -135,6 +121,8 @@ class Spider(object_ref): def start_requests(self): yield Request("https://toscrape.com/") + + .. seealso:: :ref:`spider-start` """ for item_or_request in self.start_requests(): yield item_or_request diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9115edd27..7793505da 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -194,25 +194,15 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_unsupported_output(self): - """Anything that is not a request, a seeding policy or a string (which - is assumed to be a seeding policy) is assumed to be an item, avoiding a - potentially expensive call to itemadapter.is_item, and letting instead - things fail when ItemAdapter is actually used on the corresponding - non-item object.""" + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item(), and letting + instead things fail when ItemAdapter is actually used on the + corresponding non-item object.""" with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 1 - - @defer.inlineCallbacks - def test_start_laziness(self): - settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"} - crawler = get_crawler(BrokenStartSpider, settings) - yield crawler.crawl(mockserver=self.mockserver) - assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( - 99 - ), crawler.spider.seedsseen + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_dupes(self): diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py new file mode 100644 index 000000000..62e941f49 --- /dev/null +++ b/tests/test_engine_loop.py @@ -0,0 +1,130 @@ +from collections import deque + +from twisted.internet.defer import Deferred +from twisted.trial.unittest import TestCase + +from scrapy import Request, Spider, signals +from scrapy.core.engine import ExecutionEngine +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .mockserver import MockServer +from .test_scheduler import MemoryScheduler + + +def sleep(seconds: float = 0.001): + from twisted.internet import reactor + + deferred: Deferred[None] = Deferred() + reactor.callLater(seconds, deferred.callback, None) + return maybe_deferred_to_future(deferred) + + +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_sleep(self): + """Neither asynchronous sleeps on Spider.start() nor the equivalent on + the scheduler (returning no requests while also returning True from + the has_pending_requests() method) should cause the spider to miss the + processing of any later requests.""" + seconds = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01 + + class TestSpider(Spider): + name = "test" + + async def start(self): + from twisted.internet import reactor + + yield Request("data:,a") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + + # During this time, the reactor reports having requests but + # returns None. + await sleep(seconds) + + self.crawler.engine._slot.scheduler.unpause() + + # The scheduler request is processed. + await sleep(seconds) + + yield Request("data:,c") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + + # The last start request is processed during the time until the + # delayed call below, proving that the start iteration can + # finish before a scheduler “sleep” without causing the + # scheduler to finish. + reactor.callLater(seconds, self.crawler.engine._slot.scheduler.unpause) + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + +class MockServerTestCase(TestCase): + # See the comment on the matching line above. + timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @deferred_f_from_coro_f + async def test_default_behavior(self): + """Verify the behavior that the docs claims is the default when it + comes to start request send order.""" + seconds = 0.1 + + def _url(id, delay=seconds): + return self.mockserver.url(f"/delay?n={seconds}&{id}") + + class TestSpider(Spider): + name = "test" + start_urls = [_url("a", delay=0), _url("b"), _url("d"), _url("e")] + queue = deque([Request(_url("c"))]) + + def parse(self, response): + try: + request = self.queue.popleft() + except IndexError: + pass + else: + yield request + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"CONCURRENT_REQUESTS": 2} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = [_url(letter) for letter in "abcd"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py deleted file mode 100644 index c52a2b8bd..000000000 --- a/tests/test_engine_seeding.py +++ /dev/null @@ -1,358 +0,0 @@ -from __future__ import annotations - -from collections import defaultdict, deque -from logging import ERROR - -from testfixtures import LogCapture -from twisted.trial.unittest import TestCase - -from scrapy import Request, SeedingPolicy, Spider, signals -from scrapy.core.engine import ExecutionEngine -from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future -from scrapy.utils.test import get_crawler - -from .mockserver import MockServer -from .test_spider_start import twisted_sleep - - -class MainTestCase(TestCase): - # If the test ends before the heartbeat, it may mean that the logic to - # re-schecule a new call of _start_next_requests under the right - # ciscumstances is not properly implemented, and the hearatbeat is working - # as a workaround for that issue. This is a performance issue and should - # be addressed. - # - # It could also happen that, on some CI runners, some tests (e.g. those - # below using a mock server) run too slow and proper handling overlaps with - # the heartbeat. If that is the case, it may be worth considering - # increasing the heartbeat time. It should be safe, since in most real live - # scenarios the heartbeat should never make a difference, and we may - # eventually remove the heartbeat altogether. - timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL - - @deferred_f_from_coro_f - async def test_greedy(self): - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = deque((Request("data:,b"),)) - - def enqueue_request(self, request: Request) -> bool: - self.requests.append(request) - return True - - def has_pending_requests(self) -> bool: - return bool(self.requests) - - def next_request(self) -> Request | None: - try: - return self.requests.pop() - except IndexError: - return None - - class TestSpider(Spider): - name = "test" - start_urls = ["data:,a"] - - def parse(self, response): - pass - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - @deferred_f_from_coro_f - async def test_lazy(self): - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = deque((Request("data:,a"),)) - - def enqueue_request(self, request: Request) -> bool: - self.requests.append(request) - return True - - def has_pending_requests(self) -> bool: - return bool(self.requests) - - def next_request(self) -> Request | None: - try: - return self.requests.popleft() - except IndexError: - return None - - class TestSpider(Spider): - name = "test" - start_urls = ["data:,b"] - - def parse(self, response): - pass - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - @deferred_f_from_coro_f - async def test_lazy_blocking(self): - """If the scheduler reports having requests but yields none, the lazy - policy schedules start requests.""" - - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = deque() - self.stop = False - - def enqueue_request(self, request: Request) -> bool: - self.requests.append(request) - return True - - def has_pending_requests(self) -> bool: - return not self.stop - - def next_request(self) -> Request | None: - try: - return self.requests.popleft() - except IndexError: - return None - - sleep_seconds = 0.0001 - - class TestSpider(Spider): - name = "test" - - async def start(self): - await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) - yield Request("data:,a") - await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) - self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) - await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) - yield Request("data:,c") - self.crawler.engine._slot.scheduler.stop = True - - def parse(self, response): - pass - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b", "data:,c"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - @deferred_f_from_coro_f - async def test_lazy_start_order(self): - """By default, start requests should be sent in the order in which they - are iterated.""" - - class TestSpider(Spider): - name = "test" - start_urls = ["data:,a", "data:,b", "data:,c"] - - def parse(self, response): - pass - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SEEDING_POLICY": "lazy"} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b", "data:,c"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - @deferred_f_from_coro_f - async def test_front_load(self): - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = defaultdict(deque) - - def enqueue_request(self, request: Request) -> bool: - self.requests[request.priority].append(request) - return True - - def has_pending_requests(self) -> bool: - return bool(self.requests) - - def next_request(self) -> Request | None: - if not self.requests: - return None - priority = max(self.requests) - request = self.requests[priority].popleft() - if not self.requests[priority]: - del self.requests[priority] - return request - - class TestSpider(Spider): - name = "test" - - async def start(self): - yield Request("data:,b", priority=0) - yield Request("data:,a", priority=1) - - def parse(self, response): - pass - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "front_load"} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - @deferred_f_from_coro_f - async def test_override(self): - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = defaultdict(deque) - - def enqueue_request(self, request: Request) -> bool: - self.requests[request.priority].append(request) - return True - - def has_pending_requests(self) -> bool: - return bool(self.requests) - - def next_request(self) -> Request | None: - if not self.requests: - return None - priority = max(self.requests) - request = self.requests[priority].popleft() - if not self.requests[priority]: - del self.requests[priority] - return request - - class TestSpider(Spider): - name = "test" - - async def start(self): - yield "front-load" # typo - yield SeedingPolicy.front_load - yield Request("data:,b", priority=1) - yield Request("data:,a", priority=2) - yield self.crawler.settings["SEEDING_POLICY"] - yield Request("data:,c", priority=3) - - def parse(self, response): - pass - - actual_items = [] - actual_urls = [] - - def track_item(item, response, spider): - actual_items.append(item) - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_item, signals.item_scraped) - crawler.signals.connect(track_url, signals.request_reached_downloader) - with LogCapture(level=ERROR) as log: - await maybe_deferred_to_future(crawler.crawl()) - assert len(log.records) == 1 - assert "must be valid seeding policies" in str(log.records[0]) - assert crawler.stats.get_value("finish_reason") == "finished" - assert not actual_items, ( - f"{actual_items=} should be empty, policies are not items" - ) - expected_urls = ["data:,a", "data:,b", "data:,c"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - -class MockServerTestCase(TestCase): - # See the comment on the matching line above. - timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL - # If requests are too fast, test_idle will fail because the outcome will - # match that of the lazy seeding policy. - delay = 0.2 - - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @deferred_f_from_coro_f - async def test_idle(self): - def _url(id): - return self.mockserver.url(f"/delay?n={self.delay}&{id}") - - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = deque((Request(_url("a")),)) - - def enqueue_request(self, request: Request) -> bool: - self.requests.append(request) - return True - - def has_pending_requests(self) -> bool: - return bool(self.requests) - - def next_request(self) -> Request | None: - try: - return self.requests.popleft() - except IndexError: - return None - - class TestSpider(Spider): - name = "test" - start_urls = [_url("b"), _url("d")] - queue = deque((Request(_url("c")),)) - - def parse(self, response): - try: - request = self.queue.popleft() - except IndexError: - pass - else: - yield request - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "idle"} - crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = [_url(letter) for letter in "abcd"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 1d6992a32..f90293dd3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil import tempfile from abc import ABC, abstractmethod +from collections import deque from typing import Any, NamedTuple import pytest @@ -10,7 +11,7 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider @@ -20,6 +21,38 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +class MemoryScheduler(BaseScheduler): + paused = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.queue = deque( + Request(value) if isinstance(value, str) else value + for value in getattr(self, "queue", []) + ) + + def enqueue_request(self, request: Request) -> bool: + self.queue.append(request) + return True + + def has_pending_requests(self) -> bool: + return self.paused or bool(self.queue) + + def next_request(self) -> Request | None: + if self.paused: + return None + try: + return self.queue.pop() + except IndexError: + return None + + def pause(self) -> None: + self.paused = True + + def unpause(self) -> None: + self.paused = False + + class MockEngine(NamedTuple): downloader: MockDownloader From 68f63e143051f65efbcb906a2417a9ff3f88a5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 24 Mar 2025 17:29:04 +0100 Subject: [PATCH 15/45] Fix bad ports from the main PR --- docs/topics/request-response.rst | 2 +- scrapy/http/request/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index de7840878..a98f77df5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -352,7 +352,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - async def yield_seeds(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0bfc92c6b..2b8d0ab84 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -201,7 +201,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.yield_seeds`. + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None From 5d43d91a957fd80eeb7b1b39d4487ec320ca164d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 10:59:50 +0100 Subject: [PATCH 16/45] WIP --- docs/topics/spiders.rst | 40 ++++---- tests/test_engine_loop.py | 195 +++++++++++++++++++++++++++++++++----- 2 files changed, 192 insertions(+), 43 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 2432223eb..25ce81b2e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -369,31 +369,35 @@ See `Scrapyd documentation`_. Spider start ============ -The way :meth:`~scrapy.Spider.start` works may be counterintuitive. +The way :meth:`~scrapy.Spider.start` works by default may be counterintuitive: -By default, if you do not use :ref:`await <await>` in -:meth:`~scrapy.Spider.start`, or if you instead use the -:attr:`~scrapy.Spider.start_urls` attribute, this is what happens: +#. The first 8-16 start requests are sent in the order in which they are + yielded. -- The first 8-16 start requests (based on :setting:`CONCURRENT_REQUESTS` and - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) are sent in the order in which - they are yielded. + That number depends on :setting:`CONCURRENT_REQUESTS`. + :ref:`Awaiting <await>` slow operations in :meth:`~scrapy.Spider.start` may + lower it. - .. note:: Responses may come in a different order. +#. The last start request is sent. -- The remaining start requests are sent in reverse order, and only when there - are not enough pending requests yielded from callbacks to reach the + It could be more start requests for very low response times. If so, they + are the last start requests in reverse order. + +#. The remaining start requests are also sent in reverse order, but only when + there are not enough pending requests yielded from callbacks to reach the configured concurrency. - This is because the :ref:`scheduler <topics-scheduler>`, where pending - requests are stored, uses a LIFO (last in, first out) queue by default, - configured in the :setting:`SCHEDULER_MEMORY_QUEUE` and - :setting:`SCHEDULER_DISK_QUEUE`. +.. note:: Response order is a different story: it is determined not only by + request order, but also by response time. - So, provided all pending requests have the same - :attr:`~scrapy.Request.priority`, scheduled requests are sent in reserve - order. The first few requests are sent in order only because they are sent - as soon as they are scheduled. +The reverse order is because the :ref:`scheduler <topics-scheduler>`, which +handles pending requests, stores requests with the same +:attr:`~scrapy.Request.priority` in a LIFO (last in, first out) queue by +default (see :setting:`SCHEDULER_MEMORY_QUEUE` and +:setting:`SCHEDULER_DISK_QUEUE`). The order of the first few requests is +unnaffected because they are sent as soon as they are scheduled, and the last +start requests sent before callback requests are those that can be sent before +the first callback requests are scheduled. If you need start requests to be sent before requests yielded from spider callbacks, you can set a higher priority for them. For example: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 62e941f49..fce20eda9 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -82,9 +82,6 @@ class MainTestCase(TestCase): class MockServerTestCase(TestCase): - # See the comment on the matching line above. - timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL - @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -94,37 +91,185 @@ class MockServerTestCase(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @deferred_f_from_coro_f - async def test_default_behavior(self): - """Verify the behavior that the docs claims is the default when it - comes to start request send order.""" - seconds = 0.1 + # Verify the default behavior of the engine loop as described in the docs, + # in the “Spider start” section of the page about spdiers. + # + # TODO: Check how combinations of the following parameters affect the + # behavior: response time (high/low), max concurrency (hihg/low), number of + # start requests (few/many), and number of domains (single/multiple). - def _url(id, delay=seconds): - return self.mockserver.url(f"/delay?n={seconds}&{id}") + fast_seconds = 0.001 + slow_seconds = 0.2 # increase if flaky + + @deferred_f_from_coro_f + async def _test_request_order( + self, + start_nums, + cb_nums, + settings=None, + response_seconds=None, + download_slots=1, + ): + settings = settings or {} + response_seconds = response_seconds or self.slow_seconds + + def _request(num): + url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") + meta = {"download_slot": str(num % download_slots)} + return Request(url, meta=meta) class TestSpider(Spider): name = "test" - start_urls = [_url("a", delay=0), _url("b"), _url("d"), _url("e")] - queue = deque([Request(_url("c"))]) + cb_requests = deque([_request(num) for num in cb_nums]) + + async def start(self): + for num in start_nums: + yield _request(num) def parse(self, response): - try: - request = self.queue.popleft() - except IndexError: - pass - else: - yield request + while self.cb_requests: + yield self.cb_requests.popleft() - actual_urls = [] + actual_nums = [] - def track_url(request, spider): - actual_urls.append(request.url) + def track_num(request, spider): + actual_nums.append(int(request.url.rsplit("&", maxsplit=1)[1])) - settings = {"CONCURRENT_REQUESTS": 2} crawler = get_crawler(TestSpider, settings_dict=settings) - crawler.signals.connect(track_url, signals.request_reached_downloader) + crawler.signals.connect(track_num, signals.request_reached_downloader) await maybe_deferred_to_future(crawler.crawl()) assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = [_url(letter) for letter in "abcd"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + expected_nums = sorted(start_nums + cb_nums) + assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + + # TODO: Figure out why the behavior changes when CONCURRENT_REQUESTS is + # higher than CONCURRENT_REQUESTS_PER_DOMAIN. The number of requests sent + # before callback requests seems to depend on CONCURRENT_REQUESTS and not + # in CONCURRENT_REQUESTS_PER_DOMAIN. + @deferred_f_from_coro_f + async def test_default(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 19, + 17, + 16, + 15, + 14, + 13, + 12, + 11, + 10, + ], + cb_nums=[18], + settings={"CONCURRENT_REQUESTS": 9}, + ) + ) + + @deferred_f_from_coro_f + async def test_conc1(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 4, 2], + cb_nums=[3], + settings={"CONCURRENT_REQUESTS": 1}, + ) + ) + + @deferred_f_from_coro_f + async def test_conc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 6, 4, 3], + cb_nums=[5], + settings={"CONCURRENT_REQUESTS": 2}, + ) + ) + + @deferred_f_from_coro_f + async def test_conc8(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9], + cb_nums=[17], + settings={"CONCURRENT_REQUESTS": 8}, + ) + ) + + @deferred_f_from_coro_f + async def test_conc16(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 34, + 32, + 31, + 30, + 29, + 28, + 27, + 26, + 25, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, + ], + cb_nums=[33], + settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16}, + ) + ) + + @deferred_f_from_coro_f + async def test_domains(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 6, 4, 3], + cb_nums=[5], + settings={ + "CONCURRENT_REQUESTS": 2, + "CONCURRENT_REQUESTS_PER_DOMAIN": 1, + }, + download_slots=2, + ) + ) + + @deferred_f_from_coro_f + async def test_fast(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 3, 2], + cb_nums=[4], + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=self.fast_seconds, + ) + ) From 52f931088c2bfd321f1f99a76be8849dd0f557e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 11:02:32 +0100 Subject: [PATCH 17/45] =?UTF-8?q?partial=20coroutine=20support=20=E2=86=92?= =?UTF-8?q?=20coroutine=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/coroutines.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 2a7a0c7c7..aae2950b6 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -6,8 +6,8 @@ Coroutines .. versionadded:: 2.0 -Scrapy has :ref:`partial support <coroutine-support>` for the :ref:`coroutine -syntax <async>` (i.e. ``async def``). +Scrapy :ref:`supports <coroutine-support>` the :ref:`coroutine syntax <async>` +(i.e. ``async def``). .. _coroutine-support: From 17d85b8ad7198859bf3ad1c075935ea2c38aa03d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 12:52:37 +0100 Subject: [PATCH 18/45] WIP --- docs/topics/spiders.rst | 25 ++++---- tests/test_engine_loop.py | 131 +++++++++++++++++++++++++++++++------- 2 files changed, 122 insertions(+), 34 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 25ce81b2e..bc1c58ead 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -371,21 +371,24 @@ Spider start The way :meth:`~scrapy.Spider.start` works by default may be counterintuitive: -#. The first 8-16 start requests are sent in the order in which they are - yielded. +#. First, the first 16 start requests are sent in order. - That number depends on :setting:`CONCURRENT_REQUESTS`. - :ref:`Awaiting <await>` slow operations in :meth:`~scrapy.Spider.start` may - lower it. + That number depends on :setting:`CONCURRENT_REQUESTS`. :ref:`Awaiting + <await>` slow operations in :meth:`~scrapy.Spider.start` may lower it. -#. The last start request is sent. +#. Then, the last 8 start requests are sent in reverse order. - It could be more start requests for very low response times. If so, they - are the last start requests in reverse order. + That number depends on both :setting:`CONCURRENT_REQUESTS` and + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Specifically, assuming an even + domain distribution in start requests (i.e. ABCABC, not AABBCC), it is: -#. The remaining start requests are also sent in reverse order, but only when - there are not enough pending requests yielded from callbacks to reach the - configured concurrency. + .. code-block:: python + + min(CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN * domain_count) + +#. Finally, the remaining start requests are also sent in reverse order, but + only when there are not enough pending requests yielded from callbacks to + reach the configured concurrency. .. note:: Response order is a different story: it is determined not only by request order, but also by response time. diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index fce20eda9..058deee8b 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -93,10 +93,6 @@ class MockServerTestCase(TestCase): # Verify the default behavior of the engine loop as described in the docs, # in the “Spider start” section of the page about spdiers. - # - # TODO: Check how combinations of the following parameters affect the - # behavior: response time (high/low), max concurrency (hihg/low), number of - # start requests (few/many), and number of domains (single/multiple). fast_seconds = 0.001 slow_seconds = 0.2 # increase if flaky @@ -142,10 +138,6 @@ class MockServerTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" - # TODO: Figure out why the behavior changes when CONCURRENT_REQUESTS is - # higher than CONCURRENT_REQUESTS_PER_DOMAIN. The number of requests sent - # before callback requests seems to depend on CONCURRENT_REQUESTS and not - # in CONCURRENT_REQUESTS_PER_DOMAIN. @deferred_f_from_coro_f async def test_default(self): await maybe_deferred_to_future( @@ -160,18 +152,24 @@ class MockServerTestCase(TestCase): 7, 8, 9, - 19, - 17, - 16, - 15, - 14, - 13, - 12, - 11, 10, + 11, + 12, + 13, + 14, + 15, + 16, + 26, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, ], - cb_nums=[18], - settings={"CONCURRENT_REQUESTS": 9}, + cb_nums=[25], ) ) @@ -250,14 +248,95 @@ class MockServerTestCase(TestCase): ) @deferred_f_from_coro_f - async def test_domains(self): + async def test_conc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( - start_nums=[1, 2, 6, 4, 3], - cb_nums=[5], + start_nums=[1, 2, 3, 8, 6, 5, 4], + cb_nums=[7], settings={ - "CONCURRENT_REQUESTS": 2, - "CONCURRENT_REQUESTS_PER_DOMAIN": 1, + "CONCURRENT_REQUESTS": 3, + }, + download_slots=2, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc3_dconc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 7, 5, 4], + cb_nums=[6], + settings={ + "CONCURRENT_REQUESTS": 3, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc5_dconc3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], + cb_nums=[9], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc5_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc5_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + download_slots=2, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc7_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @deferred_f_from_coro_f + async def test_tconc7_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, }, download_slots=2, ) @@ -273,3 +352,9 @@ class MockServerTestCase(TestCase): response_seconds=self.fast_seconds, ) ) + # TODO: Test how increasing concurrency behaves with fast responses. + + # TODO: Test claims: + # - :ref:`Awaiting <await>` slow operations in :meth:`~scrapy.Spider.start` may lower it. + # - If responses are very fast, it can be more than :setting:`CONCURRENT_REQUESTS`. + # - Otherwise, it can reach 16 (:setting:`CONCURRENT_REQUESTS`) From 19c910f731cd9471b09f95e5548e1206626c363c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 15:28:21 +0100 Subject: [PATCH 19/45] Complete the new loop implementation (no lazy support yet) --- docs/news.rst | 19 ++++++-- docs/topics/spiders.rst | 67 +++++++++------------------- scrapy/core/engine.py | 5 ++- scrapy/spiders/__init__.py | 2 +- tests/test_engine_loop.py | 90 ++++++++++++++++++++++++++++++-------- 5 files changed, 113 insertions(+), 70 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 37ed1e693..fd77fb6b2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,13 +11,18 @@ Scrapy VERSION (unreleased) Highlights: - Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` - (async) and changed how it is iterated by default. + (async) and changed how it is iterated. Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- By default, the iteration of start requests and items no longer stops once - there are requests in the scheduler. +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + As a result, the order in which start requests are sent may change. See + :ref:`start-requests` for details and information on how to force start + request order. - In ``scrapy.core.engine.ExecutionEngine``: @@ -66,6 +71,14 @@ New features (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) +- Start requests are now :ref:`scheduled <topics-scheduler>` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + Bug fixes ~~~~~~~~~ diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index bc1c58ead..3d6216d60 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -364,63 +364,38 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. -.. _spider-start: +.. _start-requests: -Spider start -============ +Start requests +============== -The way :meth:`~scrapy.Spider.start` works by default may be counterintuitive: +Scrapy does not try to send :meth:`~scrapy.Spider.start` requests in order. +Instead, it prioritizes reaching :setting:`CONCURRENT_REQUESTS` and +:ref:`scheduling <topics-scheduler>` start requests. -#. First, the first 16 start requests are sent in order. +To change that, override the :meth:`~scrapy.Spider.start` method to set +:attr:`Request.priority <scrapy.http.Request.priority>`. For example: - That number depends on :setting:`CONCURRENT_REQUESTS`. :ref:`Awaiting - <await>` slow operations in :meth:`~scrapy.Spider.start` may lower it. - -#. Then, the last 8 start requests are sent in reverse order. - - That number depends on both :setting:`CONCURRENT_REQUESTS` and - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Specifically, assuming an even - domain distribution in start requests (i.e. ABCABC, not AABBCC), it is: +- To send start requests before other requests: .. code-block:: python - min(CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN * domain_count) + async def start(self): + async for request in super().start(): + yield request.replace(priority=1) -#. Finally, the remaining start requests are also sent in reverse order, but - only when there are not enough pending requests yielded from callbacks to - reach the configured concurrency. +- To send start requests in order: -.. note:: Response order is a different story: it is determined not only by - request order, but also by response time. + .. code-block:: python -The reverse order is because the :ref:`scheduler <topics-scheduler>`, which -handles pending requests, stores requests with the same -:attr:`~scrapy.Request.priority` in a LIFO (last in, first out) queue by -default (see :setting:`SCHEDULER_MEMORY_QUEUE` and -:setting:`SCHEDULER_DISK_QUEUE`). The order of the first few requests is -unnaffected because they are sent as soon as they are scheduled, and the last -start requests sent before callback requests are those that can be sent before -the first callback requests are scheduled. + async def start(self): + priority = len(self.start_urls) + async for request in super().start(): + yield request.replace(priority=priority) + priority -= 1 -If you need start requests to be sent before requests yielded from spider -callbacks, you can set a higher priority for them. For example: - -.. code-block:: python - - async def start(self): - async for request in super().start(): - yield request.replace(priority=1) - -If you also need them to be sent in order, you can assign them decreasing -priority values. For example: - -.. code-block:: python - - async def start(self): - priority = len(self.start_urls) - async for request in super().start(): - yield request.replace(priority=priority) - priority -= 1 +You can also :ref:`customize the scheduler <topics-scheduler>` if you need +more control over request prioritization. .. _builtin-spiders: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6ec9780e6..63b332563 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -45,13 +45,13 @@ class _Slot: def __init__( self, close_if_idle: bool, - nextcall: CallLaterOnce[Deferred[None]], + nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() self.close_if_idle: bool = close_if_idle - self.nextcall: CallLaterOnce[Deferred[None]] = nextcall + self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) @@ -191,6 +191,7 @@ class ExecutionEngine: Items are scraped. Requests are scheduled. """ + assert self._slot is not None # typing while self._start is not None: yield self._process_next_spider_start_yield() if not self._needs_backout(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 30e197b8c..80932bc9d 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -122,7 +122,7 @@ class Spider(object_ref): def start_requests(self): yield Request("https://toscrape.com/") - .. seealso:: :ref:`spider-start` + .. seealso:: :ref:`start-requests` """ for item_or_request in self.start_requests(): yield item_or_request diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 058deee8b..0dc053208 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -81,7 +81,34 @@ class MainTestCase(TestCase): assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" -class MockServerTestCase(TestCase): +class RequestSendOrderTestCase(TestCase): + """Test the intrincacies of request send order when all start requests and + callback requests have the same priority. + + It is a very unintuitive behavior, documented as “undefined” so that we may + change it in the future without breaking the contract. + + 1. First, the first CONCURRENT_REQUESTS start requests are sent in order. + + Awaiting slow operations in Spider.start() can lower that. + + 2. Then, assuming an even domain distribution in start requests (i.e. + ABCABC, not AABBCC), the last N start requests are sent in reverse + order, where N is: + + min(CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN * domain_count) + + 3. Finally, the remaining start requests are also sent in reverse order, + but only when there are not enough pending requests yielded from + callbacks to reach the configured concurrency. + + The reverse order is because the scheduler uses a LIFO queue by default + (SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few + requests is unnaffected because they are sent as soon as they are + scheduled, and the last start requests sent before callback requests are + those that can be sent before the first callback requests are scheduled. + """ + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -91,12 +118,14 @@ class MockServerTestCase(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - # Verify the default behavior of the engine loop as described in the docs, - # in the “Spider start” section of the page about spdiers. - fast_seconds = 0.001 slow_seconds = 0.2 # increase if flaky + def _request(self, num, response_seconds, download_slots): + url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") + meta = {"download_slot": str(num % download_slots)} + return Request(url, meta=meta) + @deferred_f_from_coro_f async def _test_request_order( self, @@ -105,22 +134,26 @@ class MockServerTestCase(TestCase): settings=None, response_seconds=None, download_slots=1, + start_fn=None, ): settings = settings or {} response_seconds = response_seconds or self.slow_seconds - def _request(num): - url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") - meta = {"download_slot": str(num % download_slots)} - return Request(url, meta=meta) + if start_fn is None: + + async def start_fn(spider): + for num in start_nums: + yield self._request(num, response_seconds, download_slots) class TestSpider(Spider): name = "test" - cb_requests = deque([_request(num) for num in cb_nums]) - - async def start(self): - for num in start_nums: - yield _request(num) + cb_requests = deque( + [ + self._request(num, response_seconds, download_slots) + for num in cb_nums + ] + ) + start = start_fn def parse(self, response): while self.cb_requests: @@ -344,6 +377,8 @@ class MockServerTestCase(TestCase): @deferred_f_from_coro_f async def test_fast(self): + """Very fast responses may increase the number of start requests sent + in reverse order before the first callback request.""" await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 3, 2], @@ -352,9 +387,28 @@ class MockServerTestCase(TestCase): response_seconds=self.fast_seconds, ) ) - # TODO: Test how increasing concurrency behaves with fast responses. - # TODO: Test claims: - # - :ref:`Awaiting <await>` slow operations in :meth:`~scrapy.Spider.start` may lower it. - # - If responses are very fast, it can be more than :setting:`CONCURRENT_REQUESTS`. - # - Otherwise, it can reach 16 (:setting:`CONCURRENT_REQUESTS`) + @deferred_f_from_coro_f + async def test_await(self): + """Awaiting slow operations in Spider.start() may lower the number of + first start requests sent in order.""" + start_nums = [1, 3] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + assert len(start_nums) > 1 + for num in start_nums[:-1]: + yield self._request(num, response_seconds, download_slots) + await sleep(response_seconds * 2) + yield self._request(start_nums[-1], response_seconds, download_slots) + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=[2], + settings={"CONCURRENT_REQUESTS": 2}, + response_seconds=response_seconds, + start_fn=start, + ) + ) From 72f4877d76d070eedfac20f70b37ca0173477c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 15:39:24 +0100 Subject: [PATCH 20/45] Address feedback --- scrapy/commands/fetch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 77c318958..ef6e13de2 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -91,9 +91,8 @@ class Command(ScrapyCommand): spidercls = spidercls_for_request(spider_loader, request, spidercls) async def start(self): - yield self._request + yield request - spidercls._request = request # type: ignore[assignment,attr-defined] spidercls.start = start # type: ignore[method-assign,attr-defined] self.crawler_process.crawl(spidercls) From eddf582ba9112c3e6be9101f02895782a146c37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 16:44:27 +0100 Subject: [PATCH 21/45] Error out on uncompatible spider middlewares --- scrapy/core/engine.py | 16 ++- scrapy/core/spidermw.py | 19 +++ tests/test_engine_loop.py | 2 +- tests/test_spidermiddleware_process_start.py | 123 ++++++++++++++++++- 4 files changed, 149 insertions(+), 11 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 63b332563..7071d83db 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -93,17 +93,21 @@ class ExecutionEngine: self.spider: Spider | None = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( - crawler.settings - ) - downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) - self.downloader: Downloader = downloader_cls(crawler) - self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = ( spider_closed_callback ) self.start_time: float | None = None self._start: AsyncIterable[Any] | None = None + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) + try: + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( + crawler.settings + ) + self.downloader: Downloader = downloader_cls(crawler) + self.scraper: Scraper = Scraper(crawler) + except Exception: + self.close() + raise def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7b3826d21..2090d5506 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -69,6 +69,25 @@ class SpiderMiddlewareManager(MiddlewareManager): if hasattr(middleware, "process_start_requests") and not hasattr(middleware, "process_start") ] + modern_middlewares = [ + middleware + for middleware in middlewares + if not hasattr(middleware, "process_start_requests") + and hasattr(middleware, "process_start") + ] + if deprecated_middlewares and modern_middlewares: + raise ValueError( + "You are trying to combine spider middlewares that only " + "define the deprecated process_start_requests() method () " + "with spider middlewares that only define the " + "process_start() method (). This is not possible. You must " + "either disable or make universal 1 of those 2 sets of " + "spider middlewares. Making a spider middleware universal " + "means having it define both methods. See the release notes " + "of Scrapy VERSION for details: " + "https://docs.scrapy.org/en/VERSION/news.html" + ) + self._use_start_requests = bool(deprecated_middlewares) if self._use_start_requests: deprecated_middleware_list = ", ".join( diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 0dc053208..bf7ed9d00 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -119,7 +119,7 @@ class RequestSendOrderTestCase(TestCase): cls.mockserver.__exit__(None, None, None) fast_seconds = 0.001 - slow_seconds = 0.2 # increase if flaky + slow_seconds = 0.4 # increase if flaky def _request(self, num, response_seconds, download_slots): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 6dced5126..1913b4251 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -1,3 +1,4 @@ +import warnings from asyncio import sleep import pytest @@ -120,13 +121,21 @@ class MainTestCase(TestCase): expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] await self._test([spider_middleware], spider_cls, expected_items) + async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] + await self._test([smw1, smw2], spider_cls, expected_items) + @deferred_f_from_coro_f async def test_modern_mw_modern_spider(self): - await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) @deferred_f_from_coro_f async def test_modern_mw_universal_spider(self): - await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) @deferred_f_from_coro_f async def test_modern_mw_deprecated_spider(self): @@ -137,11 +146,15 @@ class MainTestCase(TestCase): @deferred_f_from_coro_f async def test_universal_mw_modern_spider(self): - await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) @deferred_f_from_coro_f async def test_universal_mw_universal_spider(self): - await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) @deferred_f_from_coro_f async def test_universal_mw_deprecated_spider(self): @@ -185,6 +198,108 @@ class MainTestCase(TestCase): ): await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_modern_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_universal_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_deprecated_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + async def _test_sleep(self, spider_middlewares): class TestSpider(Spider): name = "test" From 406d2fb1005aa58f443c198a68ec4dcca50437a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 17:06:55 +0100 Subject: [PATCH 22/45] Update test expectations for the default reactor --- tests/test_engine_loop.py | 281 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 268 insertions(+), 13 deletions(-) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index bf7ed9d00..3a9bcf6b9 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -1,5 +1,6 @@ from collections import deque +import pytest from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -88,6 +89,8 @@ class RequestSendOrderTestCase(TestCase): It is a very unintuitive behavior, documented as “undefined” so that we may change it in the future without breaking the contract. + For the asyncio reactor: + 1. First, the first CONCURRENT_REQUESTS start requests are sent in order. Awaiting slow operations in Spider.start() can lower that. @@ -102,6 +105,9 @@ class RequestSendOrderTestCase(TestCase): but only when there are not enough pending requests yielded from callbacks to reach the configured concurrency. + For the default Twisted reactor, step 1 sends the last CONCURRENT_REQUESTS + start requests in reverse order instead. + The reverse order is because the scheduler uses a LIFO queue by default (SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few requests is unnaffected because they are sent as soon as they are @@ -171,8 +177,11 @@ class RequestSendOrderTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + # Asyncio reactor behavior + + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_default(self): + async def test_ar_default(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[ @@ -206,8 +215,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc1(self): + async def test_ar_conc1(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 4, 2], @@ -216,8 +226,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc2(self): + async def test_ar_conc2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 6, 4, 3], @@ -226,8 +237,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc8(self): + async def test_ar_conc8(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9], @@ -236,8 +248,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc16(self): + async def test_ar_conc16(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[ @@ -280,8 +293,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_conc3_ds2(self): + async def test_ar_conc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 8, 6, 5, 4], @@ -293,8 +307,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc3_dconc2(self): + async def test_ar_tconc3_dconc2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 7, 5, 4], @@ -306,8 +321,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc5_dconc3(self): + async def test_ar_tconc5_dconc3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], @@ -319,8 +335,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc5_dconc2_ds3(self): + async def test_ar_tconc5_dconc2_ds3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], @@ -333,8 +350,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc5_dconc3_ds2(self): + async def test_ar_tconc5_dconc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], @@ -347,8 +365,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc7_dconc2_ds3(self): + async def test_ar_tconc7_dconc2_ds3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], @@ -361,8 +380,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_tconc7_dconc3_ds2(self): + async def test_ar_tconc7_dconc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], @@ -375,8 +395,9 @@ class RequestSendOrderTestCase(TestCase): ) ) + @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_fast(self): + async def test_ar_fast(self): """Very fast responses may increase the number of start requests sent in reverse order before the first callback request.""" await maybe_deferred_to_future( @@ -388,6 +409,240 @@ class RequestSendOrderTestCase(TestCase): ) ) + # Default reactor behavior + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_default(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[ + 26, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, + 16, + 15, + 14, + 13, + 12, + 11, + 10, + 9, + 8, + 7, + 6, + 5, + 4, + 3, + 2, + 1, + ], + cb_nums=[25], + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc1(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[4, 2, 1], + cb_nums=[3], + settings={"CONCURRENT_REQUESTS": 1}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[6, 4, 3, 2, 1], + cb_nums=[5], + settings={"CONCURRENT_REQUESTS": 2}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc8(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[17], + settings={"CONCURRENT_REQUESTS": 8}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc16(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[ + 34, + 32, + 31, + 30, + 29, + 28, + 27, + 26, + 25, + 24, + 23, + 22, + 21, + 20, + 19, + 18, + 17, + 16, + 15, + 14, + 13, + 12, + 11, + 10, + 9, + 8, + 7, + 6, + 5, + 4, + 3, + 2, + 1, + ], + cb_nums=[33], + settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16}, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_conc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[8, 6, 5, 4, 3, 2, 1], + cb_nums=[7], + settings={ + "CONCURRENT_REQUESTS": 3, + }, + download_slots=2, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc3_dconc2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[7, 5, 4, 3, 2, 1], + cb_nums=[6], + settings={ + "CONCURRENT_REQUESTS": 3, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc5_dconc3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[10, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[9], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc5_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc5_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[11], + settings={ + "CONCURRENT_REQUESTS": 5, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + download_slots=2, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc7_dconc2_ds3(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 2, + }, + download_slots=3, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_tconc7_dconc3_ds2(self): + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + cb_nums=[14], + settings={ + "CONCURRENT_REQUESTS": 7, + "CONCURRENT_REQUESTS_PER_DOMAIN": 3, + }, + download_slots=2, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_fast(self): + """Very fast responses may increase the number of start requests sent + in reverse order before the first callback request.""" + await maybe_deferred_to_future( + self._test_request_order( + start_nums=[3, 2, 1], + cb_nums=[4], + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=self.fast_seconds, + ) + ) + + # Behavior shared by both reactors + @deferred_f_from_coro_f async def test_await(self): """Awaiting slow operations in Spider.start() may lower the number of From 1a0b9d4f562c6cfd6839cf8557e6b01ad7e2264e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 17:16:05 +0100 Subject: [PATCH 23/45] Cover start request sorting examples in the tests --- tests/test_engine_loop.py | 76 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 3a9bcf6b9..9b7a06305 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -648,6 +648,7 @@ class RequestSendOrderTestCase(TestCase): """Awaiting slow operations in Spider.start() may lower the number of first start requests sent in order.""" start_nums = [1, 3] + cb_nums = [2] response_seconds = self.slow_seconds download_slots = 1 @@ -661,9 +662,82 @@ class RequestSendOrderTestCase(TestCase): await maybe_deferred_to_future( self._test_request_order( start_nums=start_nums, - cb_nums=[2], + cb_nums=cb_nums, settings={"CONCURRENT_REQUESTS": 2}, response_seconds=response_seconds, start_fn=start, ) ) + + # Examples from the “Start requests” section of the documentation about + # spiders. + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_ar_start_requests_first(self): + start_nums = [1, 3, 2] + cb_nums = [4] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + for num in start_nums: + request = self._request(num, response_seconds, download_slots) + yield request.replace(priority=1) + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + ) + ) + + @pytest.mark.only_not_asyncio + @deferred_f_from_coro_f + async def test_dr_start_requests_first(self): + start_nums = [3, 2, 1] + cb_nums = [4] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + for num in start_nums: + request = self._request(num, response_seconds, download_slots) + yield request.replace(priority=1) + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + ) + ) + + @deferred_f_from_coro_f + async def test_start_requests_first_sorted(self): + start_nums = [1, 2, 3] + cb_nums = [4] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + priority = len(start_nums) + for num in start_nums: + request = self._request(num, response_seconds, download_slots) + yield request.replace(priority=priority) + priority -= 1 + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + ) + ) From 97a0fed020926af464631d31bdfb354a12aea8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 17:27:58 +0100 Subject: [PATCH 24/45] Document the execution order change of non-generator start_requests, for the sake of completion --- docs/news.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index fd77fb6b2..eb7ecf8d5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -41,6 +41,10 @@ Backward-incompatible changes - In ``scrapy.core.spidermw.SpiderMiddlewareManager``, ``process_start_requests()`` has been replaced by ``process_start()``. +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler <topics-scheduler>` instance has been created. + Deprecations ~~~~~~~~~~~~ From 701f582f27af84acda85b0dd3b82965aaa8c01fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 25 Mar 2025 17:51:50 +0100 Subject: [PATCH 25/45] Implement a replacement for the removed test_graceful_crawl_error_handling --- scrapy/core/engine.py | 8 ++++---- scrapy/core/spidermw.py | 2 +- tests/test_spider_start.py | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7071d83db..e3eddd046 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging from time import time +from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -175,12 +176,11 @@ class ExecutionEngine: item_or_request = yield deferred_from_coro(self._start.__anext__()) except StopAsyncIteration: self._start = None - except Exception: + except Exception as exception: self._start = None + exception_traceback = format_exc() logger.error( - "Error while reading start items and requests", - exc_info=True, - extra={"spider": self.spider}, + f"Error while reading start items and requests: {exception}.\n{exception_traceback}" ) else: if isinstance(item_or_request, Request): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 2090d5506..bace52e4e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -390,7 +390,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_start( self, spider: Spider - ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]: + ) -> Generator[Deferred[Any], Any, AsyncIterable[Any] | None]: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: sync_start = iter(spider.start_requests()) diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 34f1d5e66..86a671f26 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,6 +1,7 @@ from asyncio import sleep import pytest +from testfixtures import LogCapture from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -151,3 +152,24 @@ class MainTestCase(TestCase): yield ITEM_A await self._test_start(start, [ITEM_A]) + + # Exceptions + + @deferred_f_from_coro_f + async def test_deprecated_non_generator_exception(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + raise RuntimeError + + with ( + LogCapture() as log, + pytest.warns( + ScrapyDeprecationWarning, + match=r"defines the deprecated start_requests\(\) method", + ), + ): + await self._test_spider(TestSpider, []) + + assert "in start_requests\n raise RuntimeError" in str(log), log From 7b89aeba0bbe3eb4e86760ddd1d6808b20f93bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 10:40:27 +0100 Subject: [PATCH 26/45] Fix tests --- scrapy/core/engine.py | 3 ++- tests/test_spider.py | 9 ++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index e3eddd046..fb2d09c79 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -180,7 +180,8 @@ class ExecutionEngine: self._start = None exception_traceback = format_exc() logger.error( - f"Error while reading start items and requests: {exception}.\n{exception_traceback}" + f"Error while reading start items and requests: {exception}.\n{exception_traceback}", + exc_info=True, ) else: if isinstance(item_or_request, Request): diff --git a/tests/test_spider.py b/tests/test_spider.py index c3d66617d..ee159a134 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -456,13 +456,8 @@ class TestCrawlSpider(TestSpider): crawler = get_crawler(TestSpider) with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: yield crawler.crawl() - log.check( - ( - "scrapy.core.engine", - "ERROR", - "Error while reading start items and requests", - ), - ) + assert "Error while reading start items and requests" in str(log) + assert "did you miss an 's'?" in str(log) class TestSitemapSpider(TestSpider): From fdba9843bb0e3057c92fc4934ee5ccddbaf0ba31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 11:26:11 +0100 Subject: [PATCH 27/45] Implement a simple scheduler_empty signal --- docs/topics/signals.rst | 11 +++++++++++ scrapy/core/engine.py | 1 + scrapy/signals.py | 1 + tests/test_signals.py | 19 +++++++++++++++++-- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 091a4c000..b13e2950f 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -131,6 +131,17 @@ engine_stopped This signal supports returning deferreds from its handlers. +scheduler_empty +~~~~~~~~~~~~~~~ + +.. signal:: scheduler_empty +.. function:: scheduler_empty() + + Sent whenever the engine asks for a pending request from the + :ref:`scheduler <topics-scheduler>` (i.e. calls its + :meth:`~scrapy.core.scheduler.BaseScheduler.next_request` method) and the + scheduler returns none. + Item signals ------------ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fb2d09c79..e1f252ef2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -229,6 +229,7 @@ class ExecutionEngine: request = self._slot.scheduler.next_request() if request is None: + self.signals.send_catch_log(signals.scheduler_empty) return None d: Deferred[Response | Request] = self._download(request) diff --git a/scrapy/signals.py b/scrapy/signals.py index 8ef0f34f0..bdeec1ba0 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -7,6 +7,7 @@ signals here without documenting them there. engine_started = object() engine_stopped = object() +scheduler_empty = object() spider_opened = object() spider_idle = object() spider_closed = object() diff --git a/tests/test_signals.py b/tests/test_signals.py index de6cf459a..663e912b7 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,8 +1,9 @@ import pytest from twisted.internet import defer -from twisted.trial import unittest +from twisted.trial.unittest import TestCase from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -20,7 +21,21 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class TestAsyncSignal(unittest.TestCase): +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_scheduler_empty(self): + crawler = get_crawler() + calls = [] + + def track_call(): + calls.append(object()) + + crawler.signals.connect(track_call, signals.scheduler_empty) + await maybe_deferred_to_future(crawler.crawl()) + assert len(calls) >= 1 + + +class MockServerTestCase(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() From 9726538cec4acbff92839ed92fa255f25d04e507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 11:30:13 +0100 Subject: [PATCH 28/45] Fix expectations for old Twisted --- tests/test_spider_start.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 86a671f26..c2975f3ed 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -2,7 +2,9 @@ from asyncio import sleep import pytest from testfixtures import LogCapture +from twisted import version as TWISTED_VERSION from twisted.internet.defer import Deferred +from twisted.python.versions import Version from twisted.trial.unittest import TestCase from scrapy import Spider, signals @@ -21,6 +23,8 @@ ASYNC_GEN_ERROR_MINIMUM_SECONDS = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.0 ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} +TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) + def twisted_sleep(seconds): from twisted.internet import reactor @@ -172,4 +176,9 @@ class MainTestCase(TestCase): ): await self._test_spider(TestSpider, []) - assert "in start_requests\n raise RuntimeError" in str(log), log + if TWISTED_KEEPS_TRACEBACKS: + assert "in start_requests\n raise RuntimeError" in str(log), log + else: + assert "in _process_next_spider_start_yield\n item_or_request =" in str( + log + ), log From 313f9de28dc39acc5941d5a089930326c3981fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 19:40:53 +0100 Subject: [PATCH 29/45] Implement lazy support --- docs/news.rst | 3 +- docs/topics/spiders.rst | 38 +- scrapy/commands/shell.py | 2 +- scrapy/core/engine.py | 77 ++-- scrapy/crawler.py | 5 +- scrapy/shell.py | 23 +- scrapy/signalmanager.py | 18 +- scrapy/utils/reactor.py | 19 +- .../__init__.py | 13 +- tests/test_downloadermiddleware.py | 44 ++- tests/test_engine.py | 24 +- tests/test_engine_loop.py | 350 +++--------------- 12 files changed, 236 insertions(+), 380 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index eb7ecf8d5..27a06358f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -22,7 +22,8 @@ Backward-incompatible changes As a result, the order in which start requests are sent may change. See :ref:`start-requests` for details and information on how to force start - request order. + request order or pause start request iteration while there are scheduled + requests. - In ``scrapy.core.engine.ExecutionEngine``: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 3d6216d60..c640609cc 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -373,16 +373,22 @@ Scrapy does not try to send :meth:`~scrapy.Spider.start` requests in order. Instead, it prioritizes reaching :setting:`CONCURRENT_REQUESTS` and :ref:`scheduling <topics-scheduler>` start requests. -To change that, override the :meth:`~scrapy.Spider.start` method to set -:attr:`Request.priority <scrapy.http.Request.priority>`. For example: +Forcing a start request order +----------------------------- + +To force a specific **request order**, override the +:meth:`~scrapy.Spider.start` method to set :attr:`Request.priority +<scrapy.http.Request.priority>`. For example: - To send start requests before other requests: .. code-block:: python async def start(self): - async for request in super().start(): - yield request.replace(priority=1) + async for item_or_request in super().start(): + if isinstance(item_or_request, Request): + item_or_request = item_or_request.replace(priority=1) + yield item_or_request - To send start requests in order: @@ -390,13 +396,33 @@ To change that, override the :meth:`~scrapy.Spider.start` method to set async def start(self): priority = len(self.start_urls) - async for request in super().start(): - yield request.replace(priority=priority) + async for item_or_request in super().start(): + if isinstance(item_or_request, Request): + item_or_request = item_or_request.replace(priority=priority) + yield item_or_request priority -= 1 You can also :ref:`customize the scheduler <topics-scheduler>` if you need more control over request prioritization. +Delaying start request iteration +-------------------------------- + +You can override the :meth:`~scrapy.Spider.start` method as follows to pause +its iteration whenever there are scheduled requests: + +.. code-block:: python + + async def start(self): + async for item_or_request in super().start(): + if self.crawler.engine.needs_backoff(): + await self.crawler.signals.wait_for(signals.scheduler_empty) + yield item_or_request + +This can help minimize the number of requests in the scheduler at any given +time, to minimize resource usage (memory or disk, depending on +:setting:`JOBDIR`). + .. _builtin-spiders: Generic Spiders diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 4c3aca605..9dabfcd9c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -85,7 +85,7 @@ class Command(ScrapyCommand): crawler._apply_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() - crawler.engine.start() + crawler.engine.start(_start_request_processing=False) self._start_crawler_thread() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index e1f252ef2..1e8d01255 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -12,7 +12,7 @@ from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import Deferred, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -20,13 +20,16 @@ from scrapy import signals from scrapy.core.scraper import Scraper, _HandleOutputDeferred from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import AsyncIterable, Callable, Generator + from collections.abc import AsyncIterable, Callable from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -99,6 +102,7 @@ class ExecutionEngine: ) self.start_time: float | None = None self._start: AsyncIterable[Any] | None = None + self._started_request_processing = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -121,22 +125,28 @@ class ExecutionEngine: ) return scheduler_cls - @inlineCallbacks - def start(self) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def start(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() - yield self.signals.send_catch_log_deferred(signal=signals.engine_started) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_started) + ) self.running = True + if _start_request_processing: + self.start_request_processing() self._closewait: Deferred[None] = Deferred() - yield self._closewait + await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" - @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + @deferred_f_from_coro_f + async def _finish_stopping_engine(_: Any) -> None: + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + ) self._closewait.callback(None) if not self.running: @@ -170,10 +180,9 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - @inlineCallbacks - def _process_next_spider_start_yield(self): + async def _process_next_spider_start_yield(self): try: - item_or_request = yield deferred_from_coro(self._start.__anext__()) + item_or_request = await self._start.__anext__() except StopAsyncIteration: self._start = None except Exception as exception: @@ -190,30 +199,37 @@ class ExecutionEngine: self.scraper.start_itemproc(item_or_request, response=None) self._slot.nextcall.schedule() - @inlineCallbacks - def _process_spider_start(self) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def start_request_processing(self) -> None: """Process start items and requests in an asynchronous loop. Items are scraped. Requests are scheduled. """ + if self._started_request_processing: + raise RuntimeError("Request processing already started") + self._started_request_processing = True + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) while self._start is not None: - yield self._process_next_spider_start_yield() - if not self._needs_backout(): + await self._process_next_spider_start_yield() + if not self.needs_backout(): self._slot.nextcall.schedule() + await self._slot.nextcall.wait() def _start_next_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: return - while not self._needs_backout(): + while not self.needs_backout(): if self._start_scheduled_request() is None: break if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() - def _needs_backout(self) -> bool: + def needs_backout(self) -> bool: assert self._slot is not None # typing assert self.scraper.slot is not None # typing return ( @@ -377,29 +393,30 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @inlineCallbacks - def open_spider( + @deferred_f_from_coro_f + async def open_spider( self, spider: Spider, close_if_idle: bool = True, - ) -> Generator[Deferred[Any], Any, None]: + ) -> None: if self._slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) + self.spider = spider nextcall = CallLaterOnce(self._start_next_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - self._start = yield self.scraper.spidermw.process_start(spider) self._slot = _Slot(close_if_idle, nextcall, scheduler) - self.spider = spider + self._start = await maybe_deferred_to_future( + self.scraper.spidermw.process_start(spider) + ) if hasattr(scheduler, "open") and (d := scheduler.open(spider)): - yield d - yield self.scraper.open_spider(spider) + await maybe_deferred_to_future(d) + await maybe_deferred_to_future(self.scraper.open_spider(spider)) assert self.crawler.stats self.crawler.stats.open_spider(spider) - yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - self._process_spider_start() - self._slot.nextcall.schedule() - self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) + ) def _spider_idle(self) -> None: """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 88d618085..749096db5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -136,6 +136,9 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + # Cannot use @deferred_f_from_coro_f because that relies on the reactor + # being installed already, which is done within _apply_settings(), inside + # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: @@ -152,7 +155,7 @@ class Crawler: self._update_root_log_handler() self.engine = self._create_engine() yield self.engine.open_spider(self.spider) - yield maybeDeferred(self.engine.start) + yield self.engine.start() except Exception: self.crawling = False if self.engine is not None: diff --git a/scrapy/shell.py b/scrapy/shell.py index 5e5e57a9a..14bc31137 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,6 +24,7 @@ from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser @@ -102,25 +103,33 @@ class Shell: # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] set_asyncio_event_loop(event_loop_path) - spider = self._open_spider(request, spider) + + def crawl_request(_): + assert self.crawler.engine is not None + self.crawler.engine.crawl(request) + + d2 = self._open_spider(request, spider) + d2.addCallback(crawl_request) + d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - assert self.crawler.engine - self.crawler.engine.crawl(request) return d - def _open_spider(self, request: Request, spider: Spider | None) -> Spider: + @deferred_f_from_coro_f + async def _open_spider(self, request: Request, spider: Spider | None) -> None: if self.spider: - return self.spider + return if spider is None: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider assert self.crawler.engine - self.crawler.engine.open_spider(spider, close_if_idle=False) + await maybe_deferred_to_future( + self.crawler.engine.open_spider(spider, close_if_idle=False) + ) + self.crawler.engine.start_request_processing() self.spider = spider - return spider def fetch( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index e106418d6..5bdc1b690 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,13 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from pydispatch import dispatcher +from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal - -if TYPE_CHECKING: - from twisted.internet.defer import Deferred +from scrapy.utils.defer import maybe_deferred_to_future class SignalManager: @@ -75,3 +74,14 @@ class SignalManager: """ kwargs.setdefault("sender", self.sender) _signal.disconnect_all(signal, **kwargs) + + async def wait_for(self, signal): + """Await the next *signal*.""" + d = Deferred() + + def handle(): + self.disconnect(handle, signal) + d.callback(None) + + self.connect(handle, signal) + await maybe_deferred_to_future(d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 099c81f0e..184a95820 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error +from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object @@ -54,6 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: DelayedCall | None = None + self._deferreds = [] def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor @@ -66,8 +68,23 @@ class CallLaterOnce(Generic[_T]): self._call.cancel() def __call__(self) -> _T: + from twisted.internet import reactor + self._call = None - return self._func(*self._a, **self._kw) + result = self._func(*self._a, **self._kw) + + for d in self._deferreds: + reactor.callLater(0, d.callback, None) + self._deferreds = [] + + return result + + async def wait(self): + from scrapy.utils.defer import maybe_deferred_to_future + + d = Deferred() + self._deferreds.append(d) + await maybe_deferred_to_future(d) def set_asyncio_event_loop_policy() -> None: diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5228f6abd..954f2c924 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -8,11 +8,16 @@ class TestCmdlineCrawlPipeline: args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) - proc.communicate() - return proc.returncode + _, stderr = proc.communicate() + return proc.returncode, stderr def test_open_spider_normally_in_pipeline(self): - assert self._execute("normal") == 0 + returncode, stderr = self._execute("normal") + assert returncode == 0 def test_exception_at_open_spider_in_pipeline(self): - assert self._execute("exception") == 1 + returncode, stderr = self._execute("exception") + assert ( + returncode == 0 + ) # An unhandled exception in a pipeline should not stop the crawl + assert b'RuntimeError("exception")' in stderr diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 5d1161750..6c061b330 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -12,6 +12,11 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_to_future, + maybe_deferred_to_future, +) from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue @@ -29,7 +34,7 @@ class TestManagerBase(TestCase): def tearDown(self): return self.crawler.engine.close_spider(self.spider) - def _download(self, request, response=None): + async def _download(self, request, response=None): """Executes downloader mw manager's download method and returns the result (Request or Response) or raise exception in case of failure. @@ -44,7 +49,7 @@ class TestManagerBase(TestCase): # catch deferred result and return the value results = [] dfd.addBoth(results.append) - self._wait(dfd) + await maybe_deferred_to_future(dfd) ret = results[0] if isinstance(ret, Failure): ret.raiseException() @@ -54,13 +59,15 @@ class TestManagerBase(TestCase): class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - def test_request_response(self): + @deferred_f_from_coro_f + async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) + ret = await self._download(req, resp) assert isinstance(ret, Response), "Non-response returned" - def test_3xx_and_invalid_gzipped_body_must_redirect(self): + @deferred_f_from_coro_f + async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -85,13 +92,14 @@ class TestDefaults(TestManagerBase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) + ret = await self._download(request=req, response=resp) assert isinstance(ret, Request), f"Not redirected: {ret!r}" assert to_bytes(ret.url) == resp.headers["Location"], ( "Not redirected to location header" ) - def test_200_and_invalid_gzipped_body_must_fail(self): + @deferred_f_from_coro_f + async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"<p>You are being redirected</p>" resp = Response( @@ -106,13 +114,14 @@ class TestDefaults(TestManagerBase): }, ) with pytest.raises(BadGzipFile): - self._download(request=req, response=resp) + await self._download(request=req, response=resp) class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - def test_download_func_not_called(self): + @deferred_f_from_coro_f + async def test_download_func_not_called(self): resp = Response("http://example.com/index.html") class ResponseMiddleware: @@ -126,7 +135,7 @@ class TestResponseFromProcessRequest(TestManagerBase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self._wait(dfd) + await maybe_deferred_to_future(dfd) assert results[0] is resp assert not download_func.called @@ -195,7 +204,8 @@ class TestProcessExceptionInvalidOutput(TestManagerBase): class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" - def test_deferred(self): + @deferred_f_from_coro_f + async def test_deferred(self): resp = Response("http://example.com/index.html") class DeferredMiddleware: @@ -214,7 +224,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self._wait(dfd) + await maybe_deferred_to_future(dfd) assert results[0] is resp assert not download_func.called @@ -224,7 +234,8 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - def test_asyncdef(self): + @deferred_f_from_coro_f + async def test_asyncdef(self): resp = Response("http://example.com/index.html") class CoroMiddleware: @@ -238,13 +249,14 @@ class TestMiddlewareUsingCoro(TestManagerBase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self._wait(dfd) + await maybe_deferred_to_future(dfd) assert results[0] is resp assert not download_func.called @pytest.mark.only_asyncio - def test_asyncdef_asyncio(self): + @deferred_f_from_coro_f + async def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") class CoroMiddleware: @@ -258,7 +270,7 @@ class TestMiddlewareUsingCoro(TestManagerBase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self._wait(dfd) + await deferred_to_future(dfd) assert results[0] is resp assert not download_func.called diff --git a/tests/test_engine.py b/tests/test_engine.py index 4dc77e438..381cb5bcf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -150,7 +150,6 @@ class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): - self.spider = None self.respplug = [] self.reqplug = [] self.reqdropped = [] @@ -191,7 +190,6 @@ class CrawlerRun: self.response_downloaded, signals.response_downloaded ) self.crawler.crawl(start_urls=start_urls) - self.spider = self.crawler.spider self.deferred = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) @@ -297,7 +295,7 @@ class TestEngineBase(unittest.TestCase): assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: assert failure.value.__class__ is ZeroDivisionError - assert spider == run.spider + assert spider == run.crawler.spider assert item["url"] == response.url if "item1.html" in item["url"]: @@ -378,11 +376,14 @@ class TestEngineBase(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - assert {"spider": run.spider} == run.signals_caught[signals.spider_opened] - assert {"spider": run.spider} == run.signals_caught[signals.spider_idle] - assert {"spider": run.spider, "reason": "finished"} == run.signals_caught[ - signals.spider_closed + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] class TestEngine(TestEngineBase): @@ -420,9 +421,10 @@ class TestEngine(TestEngineBase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - assert {"spider": run.spider, "reason": "custom_reason"} == run.signals_caught[ - signals.spider_closed - ] + assert { + "spider": run.crawler.spider, + "reason": "custom_reason", + } == run.signals_caught[signals.spider_closed] @defer.inlineCallbacks def test_close_downloader(self): @@ -472,7 +474,7 @@ class TestEngine(TestEngineBase): finally: timer.cancel() - assert b"Traceback" not in stderr + assert b"Traceback" not in stderr, stderr def test_request_scheduled_signal(caplog): diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 9b7a06305..b2dd81b7f 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -1,6 +1,5 @@ from collections import deque -import pytest from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -13,12 +12,12 @@ from .mockserver import MockServer from .test_scheduler import MemoryScheduler -def sleep(seconds: float = 0.001): +async def sleep(seconds: float = 0.001) -> None: from twisted.internet import reactor deferred: Deferred[None] = Deferred() reactor.callLater(seconds, deferred.callback, None) - return maybe_deferred_to_future(deferred) + await maybe_deferred_to_future(deferred) class MainTestCase(TestCase): @@ -87,9 +86,7 @@ class RequestSendOrderTestCase(TestCase): callback requests have the same priority. It is a very unintuitive behavior, documented as “undefined” so that we may - change it in the future without breaking the contract. - - For the asyncio reactor: + change it in the future without breaking the contract: 1. First, the first CONCURRENT_REQUESTS start requests are sent in order. @@ -105,16 +102,16 @@ class RequestSendOrderTestCase(TestCase): but only when there are not enough pending requests yielded from callbacks to reach the configured concurrency. - For the default Twisted reactor, step 1 sends the last CONCURRENT_REQUESTS - start requests in reverse order instead. - The reverse order is because the scheduler uses a LIFO queue by default (SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few requests is unnaffected because they are sent as soon as they are - scheduled, and the last start requests sent before callback requests are - those that can be sent before the first callback requests are scheduled. + scheduled. The last start requests sent before callback requests are those + that can be sent before the first callback requests are scheduled. """ + # Error out if any tests relies on the heartbeat. + timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -177,11 +174,8 @@ class RequestSendOrderTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" - # Asyncio reactor behavior - - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_default(self): + async def test_default(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[ @@ -215,9 +209,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_conc1(self): + async def test_conc1(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 4, 2], @@ -226,9 +219,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_conc2(self): + async def test_conc2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 6, 4, 3], @@ -237,9 +229,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_conc8(self): + async def test_conc8(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9], @@ -248,9 +239,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_conc16(self): + async def test_conc16(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[ @@ -293,9 +283,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_conc3_ds2(self): + async def test_conc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 8, 6, 5, 4], @@ -307,9 +296,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_tconc3_dconc2(self): + async def test_tconc3_dconc2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 7, 5, 4], @@ -321,9 +309,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_tconc5_dconc3(self): + async def test_tconc5_dconc3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], @@ -335,9 +322,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_tconc5_dconc2_ds3(self): + async def test_tconc5_dconc2_ds3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], @@ -350,9 +336,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_tconc5_dconc3_ds2(self): + async def test_tconc5_dconc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], @@ -365,9 +350,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_tconc7_dconc2_ds3(self): + async def test_tconc7_dconc2_ds3(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], @@ -380,9 +364,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_tconc7_dconc3_ds2(self): + async def test_tconc7_dconc3_ds2(self): await maybe_deferred_to_future( self._test_request_order( start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], @@ -395,9 +378,8 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_fast(self): + async def test_fast(self): """Very fast responses may increase the number of start requests sent in reverse order before the first callback request.""" await maybe_deferred_to_future( @@ -409,240 +391,6 @@ class RequestSendOrderTestCase(TestCase): ) ) - # Default reactor behavior - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_default(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[ - 26, - 24, - 23, - 22, - 21, - 20, - 19, - 18, - 17, - 16, - 15, - 14, - 13, - 12, - 11, - 10, - 9, - 8, - 7, - 6, - 5, - 4, - 3, - 2, - 1, - ], - cb_nums=[25], - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_conc1(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[4, 2, 1], - cb_nums=[3], - settings={"CONCURRENT_REQUESTS": 1}, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_conc2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[6, 4, 3, 2, 1], - cb_nums=[5], - settings={"CONCURRENT_REQUESTS": 2}, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_conc8(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[18, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - cb_nums=[17], - settings={"CONCURRENT_REQUESTS": 8}, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_conc16(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[ - 34, - 32, - 31, - 30, - 29, - 28, - 27, - 26, - 25, - 24, - 23, - 22, - 21, - 20, - 19, - 18, - 17, - 16, - 15, - 14, - 13, - 12, - 11, - 10, - 9, - 8, - 7, - 6, - 5, - 4, - 3, - 2, - 1, - ], - cb_nums=[33], - settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16}, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_conc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[8, 6, 5, 4, 3, 2, 1], - cb_nums=[7], - settings={ - "CONCURRENT_REQUESTS": 3, - }, - download_slots=2, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_tconc3_dconc2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[7, 5, 4, 3, 2, 1], - cb_nums=[6], - settings={ - "CONCURRENT_REQUESTS": 3, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_tconc5_dconc3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[10, 8, 7, 6, 5, 4, 3, 2, 1], - cb_nums=[9], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_tconc5_dconc2_ds3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - cb_nums=[11], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - download_slots=3, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_tconc5_dconc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - cb_nums=[11], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - download_slots=2, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_tconc7_dconc2_ds3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - cb_nums=[14], - settings={ - "CONCURRENT_REQUESTS": 7, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - download_slots=3, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_tconc7_dconc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], - cb_nums=[14], - settings={ - "CONCURRENT_REQUESTS": 7, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - download_slots=2, - ) - ) - - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_fast(self): - """Very fast responses may increase the number of start requests sent - in reverse order before the first callback request.""" - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[3, 2, 1], - cb_nums=[4], - settings={"CONCURRENT_REQUESTS": 1}, - response_seconds=self.fast_seconds, - ) - ) - - # Behavior shared by both reactors - @deferred_f_from_coro_f async def test_await(self): """Awaiting slow operations in Spider.start() may lower the number of @@ -672,9 +420,8 @@ class RequestSendOrderTestCase(TestCase): # Examples from the “Start requests” section of the documentation about # spiders. - @pytest.mark.only_asyncio @deferred_f_from_coro_f - async def test_ar_start_requests_first(self): + async def test_start_requests_first(self): start_nums = [1, 3, 2] cb_nums = [4] response_seconds = self.slow_seconds @@ -695,29 +442,6 @@ class RequestSendOrderTestCase(TestCase): ) ) - @pytest.mark.only_not_asyncio - @deferred_f_from_coro_f - async def test_dr_start_requests_first(self): - start_nums = [3, 2, 1] - cb_nums = [4] - response_seconds = self.slow_seconds - download_slots = 1 - - async def start(spider): - for num in start_nums: - request = self._request(num, response_seconds, download_slots) - yield request.replace(priority=1) - - await maybe_deferred_to_future( - self._test_request_order( - start_nums=start_nums, - cb_nums=cb_nums, - settings={"CONCURRENT_REQUESTS": 1}, - response_seconds=response_seconds, - start_fn=start, - ) - ) - @deferred_f_from_coro_f async def test_start_requests_first_sorted(self): start_nums = [1, 2, 3] @@ -741,3 +465,33 @@ class RequestSendOrderTestCase(TestCase): start_fn=start, ) ) + + @deferred_f_from_coro_f + async def test_lazy(self): + start_nums = [1, 2, 4] + cb_nums = [3] + response_seconds = self.slow_seconds + download_slots = 1 + + async def start(spider): + for num in start_nums: + if spider.crawler.engine.needs_backout(): + await spider.crawler.signals.wait_for(signals.scheduler_empty) + request = self._request(num, response_seconds, download_slots) + yield request + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={ + "CONCURRENT_REQUESTS": 1, + # Without the lazy approach, using the FIFO queue would + # yield a different result, with start requests not being + # sorted. + "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.FifoMemoryQueue", + }, + response_seconds=response_seconds, + start_fn=start, + ) + ) From bfb720764ba60114af52bfb089f1247413df7198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 19:44:29 +0100 Subject: [PATCH 30/45] Restore old issue title reference --- docs/contributing.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 34e92d8f0..f5c1c74b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -228,9 +228,10 @@ with a name of the branch you want to create locally). See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. -E.g. For bug #411: "Scrapy hangs if an exception raises in start" prefer -"Fix hanging when exception occurs in start (#411)" instead of "Fix for -#411". Complete titles make it easy to skim through the issue tracker. +E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" +prefer "Fix hanging when exception occurs in start_requests (#411)" +instead of "Fix for #411". Complete titles make it easy to skim through +the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits from functional changes. This will make pull From d2bb2579f7f98ffba9a2bac5df8a3b27bebbfba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 20:49:44 +0100 Subject: [PATCH 31/45] Clean up --- scrapy/commands/check.py | 4 +--- scrapy/core/engine.py | 25 +++++++++++++++++-------- scrapy/core/spidermw.py | 26 +++++++++----------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 8a3cf5078..56dc1ea55 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -81,14 +81,12 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader async def start(self): - for request in conman.from_spider(self, self._result): + for request in conman.from_spider(self, result): yield request with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - - spidercls._result = result # type: ignore[assignment,attr-defined,method-assign,return-value] spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1e8d01255..b08de6149 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -180,7 +180,12 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - async def _process_next_spider_start_yield(self): + async def _process_start_next(self): + """Processes the next item or request from Spider.start(). + + If a request, it is scheduled. If an item, it is sent to item + pipelines. + """ try: item_or_request = await self._start.__anext__() except StopAsyncIteration: @@ -201,24 +206,28 @@ class ExecutionEngine: @deferred_f_from_coro_f async def start_request_processing(self) -> None: - """Process start items and requests in an asynchronous loop. - - Items are scraped. Requests are scheduled. - """ + """Starts consuming Spider.start() output and sending scheduled + requests.""" if self._started_request_processing: raise RuntimeError("Request processing already started") self._started_request_processing = True + # Starts the processing of scheduled requests, as well as a periodic + # call to that processing method for scenarios where the scheduler + # reports having pending requests but returns none. assert self._slot is not None # typing self._slot.nextcall.schedule() self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + while self._start is not None: - await self._process_next_spider_start_yield() + await self._process_start_next() if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. self._slot.nextcall.schedule() await self._slot.nextcall.wait() - def _start_next_requests(self) -> None: + def _start_scheduled_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: return @@ -403,7 +412,7 @@ class ExecutionEngine: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) self.spider = spider - nextcall = CallLaterOnce(self._start_next_requests) + nextcall = CallLaterOnce(self._start_scheduled_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) self._slot = _Slot(close_if_idle, nextcall, scheduler) self._start = await maybe_deferred_to_future( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index bace52e4e..f58d96fd7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging from collections.abc import AsyncIterable, Callable, Iterable -from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction +from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from warnings import warn @@ -387,20 +387,20 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd2.addErrback(process_spider_exception) return dfd2 - @inlineCallbacks - def process_start( - self, spider: Spider - ) -> Generator[Deferred[Any], Any, AsyncIterable[Any] | None]: + @deferred_f_from_coro_f + async def process_start(self, spider: Spider) -> AsyncIterable[Any] | None: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: sync_start = iter(spider.start_requests()) - sync_start = yield self._process_chain( - "process_start_requests", sync_start, spider + sync_start = await maybe_deferred_to_future( + self._process_chain("process_start_requests", sync_start, spider) ) start = as_async_generator(sync_start) else: - start = yield self._iter_seeds(spider) - start = yield self._process_chain("process_start", start) + start = spider.start() + start = await maybe_deferred_to_future( + self._process_chain("process_start", start) + ) return start def _check_deprecated_start_requests_use(self, spider: Spider): @@ -473,14 +473,6 @@ class SpiderMiddlewareManager(MiddlewareManager): f"https://docs.scrapy.org/en/VERSION/news.html" ) - @staticmethod - def _iter_seeds(spider: Spider): - fn = spider.start - if isasyncgenfunction(fn): - return fn().__aiter__() - assert iscoroutinefunction(fn) - return deferred_from_coro(fn()) - # This method is only needed until _async compatibility methods are removed. @staticmethod def _get_async_method_pair( From 4a23215ce34b6fdad4a9229611e21ab7f9e169be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 21:13:18 +0100 Subject: [PATCH 32/45] Address typing issues --- scrapy/commands/bench.py | 4 ++-- scrapy/core/engine.py | 4 ++-- scrapy/core/spidermw.py | 6 +++--- scrapy/spiders/__init__.py | 4 ++-- scrapy/spiders/init.py | 4 ++-- scrapy/spiders/sitemap.py | 4 ++-- scrapy/utils/reactor.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 4cd5d640a..96bb1ae84 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,7 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import AsyncIterable + from collections.abc import AsyncIterator class Command(ScrapyCommand): @@ -59,7 +59,7 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" yield scrapy.Request(url, dont_filter=True) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b08de6149..d2fe6a813 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -29,7 +29,7 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import AsyncIterable, Callable + from collections.abc import AsyncIterator, Callable from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -101,7 +101,7 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._start: AsyncIterable[Any] | None = None + self._start: AsyncIterator[Any] | None = None self._started_request_processing = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index f58d96fd7..f8946b221 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, Callable, Iterable +from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -388,14 +388,14 @@ class SpiderMiddlewareManager(MiddlewareManager): return dfd2 @deferred_f_from_coro_f - async def process_start(self, spider: Spider) -> AsyncIterable[Any] | None: + async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: sync_start = iter(spider.start_requests()) sync_start = await maybe_deferred_to_future( self._process_chain("process_start_requests", sync_start, spider) ) - start = as_async_generator(sync_start) + start: AsyncIterator[Any] = as_async_generator(sync_start) else: start = spider.start() start = await maybe_deferred_to_future( diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 80932bc9d..acd16ee41 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,7 +15,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable from twisted.internet.defer import Deferred @@ -78,7 +78,7 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. .. versionadded:: VERSION diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 3e74640eb..ca5f08e98 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from collections.abc import AsyncIterable, Iterable +from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -29,7 +29,7 @@ class InitSpider(Spider): stacklevel=2, ) - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: for item_or_request in self.start_requests(): yield item_or_request diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 6db10c1ab..2813a32a0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import AsyncIterable, Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,7 +53,7 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: for item_or_request in self.start_requests(): yield item_or_request diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 184a95820..0002aef1a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: DelayedCall | None = None - self._deferreds = [] + self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor From 476234b07a46b896cf9bd2edd4e8b15e7902d7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 22:28:02 +0100 Subject: [PATCH 33/45] Fix tests --- tests/__init__.py | 6 ++++++ tests/test_cmdline_crawl_with_pipeline/__init__.py | 12 ++++++++---- tests/test_engine_loop.py | 2 +- tests/test_spider_start.py | 11 +---------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index cd52ade58..ccfabb0da 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,6 +8,9 @@ import os import socket from pathlib import Path +from twisted import version as TWISTED_VERSION +from twisted.python.versions import Version + # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server # (e.g. because urllib does not fully understand the proxy spec) @@ -30,3 +33,6 @@ except socket.gaierror: def get_testdata(*paths: str) -> bytes: """Return test data""" return Path(tests_datadir, *paths).read_bytes() + + +TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 954f2c924..5006e3689 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,6 +2,8 @@ import sys from pathlib import Path from subprocess import PIPE, Popen +from .. import TWISTED_KEEPS_TRACEBACKS + class TestCmdlineCrawlPipeline: def _execute(self, spname): @@ -17,7 +19,9 @@ class TestCmdlineCrawlPipeline: def test_exception_at_open_spider_in_pipeline(self): returncode, stderr = self._execute("exception") - assert ( - returncode == 0 - ) # An unhandled exception in a pipeline should not stop the crawl - assert b'RuntimeError("exception")' in stderr + # An unhandled exception in a pipeline should not stop the crawl + assert returncode == 0 + if TWISTED_KEEPS_TRACEBACKS: + assert b'RuntimeError("exception")' in stderr + else: + assert b"RuntimeError: exception" in stderr diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index b2dd81b7f..845bbad3f 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -122,7 +122,7 @@ class RequestSendOrderTestCase(TestCase): cls.mockserver.__exit__(None, None, None) fast_seconds = 0.001 - slow_seconds = 0.4 # increase if flaky + slow_seconds = 1 # increase if flaky def _request(self, num, response_seconds, download_slots): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index c2975f3ed..006701220 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -2,9 +2,7 @@ from asyncio import sleep import pytest from testfixtures import LogCapture -from twisted import version as TWISTED_VERSION from twisted.internet.defer import Deferred -from twisted.python.versions import Version from twisted.trial.unittest import TestCase from scrapy import Spider, signals @@ -23,8 +21,6 @@ ASYNC_GEN_ERROR_MINIMUM_SECONDS = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.0 ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} -TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) - def twisted_sleep(seconds): from twisted.internet import reactor @@ -176,9 +172,4 @@ class MainTestCase(TestCase): ): await self._test_spider(TestSpider, []) - if TWISTED_KEEPS_TRACEBACKS: - assert "in start_requests\n raise RuntimeError" in str(log), log - else: - assert "in _process_next_spider_start_yield\n item_or_request =" in str( - log - ), log + assert "in start_requests\n raise RuntimeError" in str(log) From 7668a693e8c5f5a0c1710cd7d8b5470c6e7f252e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 23:07:39 +0100 Subject: [PATCH 34/45] Do not install the requested event loop if it happens to match the existing one --- scrapy/utils/reactor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 0002aef1a..9c2754394 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -131,8 +131,10 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) + event_loop = _get_asyncio_event_loop() + if not isinstance(event_loop, event_loop_class): + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: try: with catch_warnings(): From 49b839b270253dcc547e0a69ab5c20eb52bbcfc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 26 Mar 2025 23:28:29 +0100 Subject: [PATCH 35/45] Reliability improvements --- scrapy/core/engine.py | 4 +++- tests/test_engine_loop.py | 5 +---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d2fe6a813..4ef996464 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -198,6 +198,8 @@ class ExecutionEngine: exc_info=True, ) else: + if not self.spider: + return # spider already closed if isinstance(item_or_request, Request): self.crawl(item_or_request) else: @@ -219,7 +221,7 @@ class ExecutionEngine: self._slot.nextcall.schedule() self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) - while self._start is not None: + while self._start and self.spider: await self._process_start_next() if not self.needs_backout(): # Give room for the outcome of self._process_start_next() to be diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 845bbad3f..a0d337955 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -109,9 +109,6 @@ class RequestSendOrderTestCase(TestCase): that can be sent before the first callback requests are scheduled. """ - # Error out if any tests relies on the heartbeat. - timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL - @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -122,7 +119,7 @@ class RequestSendOrderTestCase(TestCase): cls.mockserver.__exit__(None, None, None) fast_seconds = 0.001 - slow_seconds = 1 # increase if flaky + slow_seconds = 2 # increase if flaky def _request(self, num, response_seconds, download_slots): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") From 581c969ead4c81d5e345403d9d4f36b3997fe48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 00:11:30 +0100 Subject: [PATCH 36/45] Remove tests for undefined behavior --- tests/test_engine_loop.py | 261 ++------------------------------------ 1 file changed, 8 insertions(+), 253 deletions(-) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index a0d337955..8ef62d018 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -4,7 +4,6 @@ from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from scrapy import Request, Spider, signals -from scrapy.core.engine import ExecutionEngine from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler @@ -27,7 +26,7 @@ class MainTestCase(TestCase): the scheduler (returning no requests while also returning True from the has_pending_requests() method) should cause the spider to miss the processing of any later requests.""" - seconds = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01 + seconds = 2 class TestSpider(Spider): name = "test" @@ -109,6 +108,8 @@ class RequestSendOrderTestCase(TestCase): that can be sent before the first callback requests are scheduled. """ + seconds = 0.1 # increase if flaky + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -116,10 +117,7 @@ class RequestSendOrderTestCase(TestCase): @classmethod def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - fast_seconds = 0.001 - slow_seconds = 2 # increase if flaky + cls.mockserver.__exit__(None, None, None) # increase if flaky def _request(self, num, response_seconds, download_slots): url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") @@ -137,7 +135,7 @@ class RequestSendOrderTestCase(TestCase): start_fn=None, ): settings = settings or {} - response_seconds = response_seconds or self.slow_seconds + response_seconds = response_seconds or self.seconds if start_fn is None: @@ -171,249 +169,6 @@ class RequestSendOrderTestCase(TestCase): expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" - @deferred_f_from_coro_f - async def test_default(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 26, - 24, - 23, - 22, - 21, - 20, - 19, - 18, - 17, - ], - cb_nums=[25], - ) - ) - - @deferred_f_from_coro_f - async def test_conc1(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 4, 2], - cb_nums=[3], - settings={"CONCURRENT_REQUESTS": 1}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 6, 4, 3], - cb_nums=[5], - settings={"CONCURRENT_REQUESTS": 2}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc8(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 6, 7, 8, 18, 16, 15, 14, 13, 12, 11, 10, 9], - cb_nums=[17], - settings={"CONCURRENT_REQUESTS": 8}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc16(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 34, - 32, - 31, - 30, - 29, - 28, - 27, - 26, - 25, - 24, - 23, - 22, - 21, - 20, - 19, - 18, - 17, - ], - cb_nums=[33], - settings={"CONCURRENT_REQUESTS_PER_DOMAIN": 16}, - ) - ) - - @deferred_f_from_coro_f - async def test_conc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 8, 6, 5, 4], - cb_nums=[7], - settings={ - "CONCURRENT_REQUESTS": 3, - }, - download_slots=2, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc3_dconc2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 7, 5, 4], - cb_nums=[6], - settings={ - "CONCURRENT_REQUESTS": 3, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc5_dconc3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 10, 8, 7, 6], - cb_nums=[9], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc5_dconc2_ds3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], - cb_nums=[11], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - download_slots=3, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc5_dconc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 12, 10, 9, 8, 7, 6], - cb_nums=[11], - settings={ - "CONCURRENT_REQUESTS": 5, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - download_slots=2, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc7_dconc2_ds3(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], - cb_nums=[14], - settings={ - "CONCURRENT_REQUESTS": 7, - "CONCURRENT_REQUESTS_PER_DOMAIN": 2, - }, - download_slots=3, - ) - ) - - @deferred_f_from_coro_f - async def test_tconc7_dconc3_ds2(self): - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 2, 3, 4, 5, 6, 7, 15, 13, 12, 11, 10, 9, 8], - cb_nums=[14], - settings={ - "CONCURRENT_REQUESTS": 7, - "CONCURRENT_REQUESTS_PER_DOMAIN": 3, - }, - download_slots=2, - ) - ) - - @deferred_f_from_coro_f - async def test_fast(self): - """Very fast responses may increase the number of start requests sent - in reverse order before the first callback request.""" - await maybe_deferred_to_future( - self._test_request_order( - start_nums=[1, 3, 2], - cb_nums=[4], - settings={"CONCURRENT_REQUESTS": 1}, - response_seconds=self.fast_seconds, - ) - ) - - @deferred_f_from_coro_f - async def test_await(self): - """Awaiting slow operations in Spider.start() may lower the number of - first start requests sent in order.""" - start_nums = [1, 3] - cb_nums = [2] - response_seconds = self.slow_seconds - download_slots = 1 - - async def start(spider): - assert len(start_nums) > 1 - for num in start_nums[:-1]: - yield self._request(num, response_seconds, download_slots) - await sleep(response_seconds * 2) - yield self._request(start_nums[-1], response_seconds, download_slots) - - await maybe_deferred_to_future( - self._test_request_order( - start_nums=start_nums, - cb_nums=cb_nums, - settings={"CONCURRENT_REQUESTS": 2}, - response_seconds=response_seconds, - start_fn=start, - ) - ) - # Examples from the “Start requests” section of the documentation about # spiders. @@ -421,7 +176,7 @@ class RequestSendOrderTestCase(TestCase): async def test_start_requests_first(self): start_nums = [1, 3, 2] cb_nums = [4] - response_seconds = self.slow_seconds + response_seconds = self.seconds download_slots = 1 async def start(spider): @@ -443,7 +198,7 @@ class RequestSendOrderTestCase(TestCase): async def test_start_requests_first_sorted(self): start_nums = [1, 2, 3] cb_nums = [4] - response_seconds = self.slow_seconds + response_seconds = self.seconds download_slots = 1 async def start(spider): @@ -467,7 +222,7 @@ class RequestSendOrderTestCase(TestCase): async def test_lazy(self): start_nums = [1, 2, 4] cb_nums = [3] - response_seconds = self.slow_seconds + response_seconds = self.seconds download_slots = 1 async def start(spider): From 6a378bb94a509bbba728fab6e319ab58906a68b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 12:05:20 +0100 Subject: [PATCH 37/45] Remove enum-tools --- docs/conf.py | 1 - docs/requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6985e38fe..1167ce050 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,6 @@ author = "Scrapy developers" # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "enum_tools.autoenum", "hoverxref.extension", "notfound.extension", "scrapydocs", diff --git a/docs/requirements.txt b/docs/requirements.txt index 63243fcf3..103fb08d6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,3 @@ -enum-tools[sphinx]==0.12.0 sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 From 5e18a132daeeec89552adfd8a044212f22195a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 14:03:39 +0100 Subject: [PATCH 38/45] Clean up from self-review --- docs/intro/tutorial.rst | 2 +- docs/news.rst | 33 +++++++++++---- docs/topics/api.rst | 6 +++ docs/topics/coroutines.rst | 10 +++-- docs/topics/signals.rst | 2 + docs/topics/spider-middleware.rst | 2 +- docs/topics/spiders.rst | 31 ++++++++++++++ scrapy/commands/parse.py | 4 +- scrapy/core/engine.py | 5 +++ scrapy/core/scraper.py | 16 +++---- scrapy/core/spidermw.py | 44 ++++++++++---------- scrapy/signalmanager.py | 5 ++- scrapy/spidermiddlewares/depth.py | 6 +-- scrapy/spidermiddlewares/offsite.py | 6 +-- scrapy/spidermiddlewares/referer.py | 6 +-- scrapy/spidermiddlewares/urllength.py | 6 +-- scrapy/spiders/crawl.py | 6 +-- scrapy/utils/asyncgen.py | 8 ++-- scrapy/utils/defer.py | 10 ++--- scrapy/utils/python.py | 15 +++---- tests/test_engine_loop.py | 34 ++------------- tests/test_spider_start.py | 14 ++----- tests/test_spidermiddleware.py | 25 +++++------ tests/test_spidermiddleware_process_start.py | 6 +-- 24 files changed, 166 insertions(+), 136 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 71b420be6..c4e04364b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -172,7 +172,7 @@ request (in this case, the ``parse`` method) with a A shortcut to the ``start`` method ----------------------------------------- +---------------------------------- Instead of implementing a :meth:`~scrapy.Spider.start` method that yields :class:`~scrapy.Request` objects from URLs, you can define a diff --git a/docs/news.rst b/docs/news.rst index 27a06358f..0add34a81 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -22,14 +22,19 @@ Backward-incompatible changes As a result, the order in which start requests are sent may change. See :ref:`start-requests` for details and information on how to force start - request order or pause start request iteration while there are scheduled - requests. + request order or :ref:`pause start request iteration while there are + scheduled requests <start-requests-lazy>`. + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware <topics-spider-middleware>` no longer stops the + crawl. - In ``scrapy.core.engine.ExecutionEngine``: - - The second parameter of ``open_spider()``, ``start_requests()``, has - been removed. The starting requests are determined by the ``spider`` - parameter instead (see :meth:`~scrapy.Spider.start`). + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). - The ``slot`` attribute has been renamed to ``_slot`` and should not be used. @@ -68,7 +73,8 @@ New features - You can now yield the start requests and items of a spider from the :meth:`~scrapy.Spider.start` spider method and from the :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider - middleware method, both asynchronous generators. + middleware method, both :term:`asynchronous generators <python:asynchronous + generator>`. This makes it possible to use asynchronous code to generate those start requests and items, e.g. reading them from a queue service or database @@ -84,6 +90,15 @@ New features (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) +- :class:`Crawler.signals <scrapy.signalmanager.SignalManager>` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + +- Added a new :signal:`scheduler_empty` signal. + +- Exposed a new method of :class:`Crawler.engine + <scrapy.core.engine.ExecutionEngine>`: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + Bug fixes ~~~~~~~~~ @@ -398,9 +413,9 @@ New features - ``scrapy.Spider.start_requests()`` can now yield items. (:issue:`5289`, :issue:`6417`) - .. note:: Some third-party spider middlewares may need to be updated for - Scrapy 2.12 support before you can use them in combination with the - ability to yield items from ``start_requests()``. + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. - Added a new :class:`~scrapy.http.Response` subclass, :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 5a00fd570..8e8f3a0c9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -280,3 +280,9 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index aae2950b6..18aa8c68a 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -18,7 +18,8 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- The :meth:`~scrapy.spiders.Spider.start` spider method. +- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. .. versionadded: VERSION @@ -46,8 +47,8 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method of :ref:`spider middlewares <topics-spider-middleware>`. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. + If defined as a coroutine, it must be an :term:`asynchronous generator`. + The input ``result`` parameter is an :term:`asynchronous iterable`. See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. @@ -55,7 +56,8 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method - of :ref:`spider middlewares <custom-spider-middleware>`. + of :ref:`spider middlewares <custom-spider-middleware>`, which *must* be + defined as an :term:`asynchronous generator`. .. versionadded:: VERSION diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index b13e2950f..66cb87fc5 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -142,6 +142,8 @@ scheduler_empty :meth:`~scrapy.core.scheduler.BaseScheduler.next_request` method) and the scheduler returns none. + See :ref:`start-requests-lazy` for an example. + Item signals ------------ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0baf094bc..5b523893a 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,7 +70,7 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_start(start: AsyncIterable[Any], /) -> AsyncIterable[Any] + .. method:: process_start(start: AsyncIterator[Any], /) -> AsyncIterator[Any] :async: Iterate over the output of :meth:`~scrapy.Spider.start` or that diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c640609cc..ae80f347d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -373,6 +373,35 @@ Scrapy does not try to send :meth:`~scrapy.Spider.start` requests in order. Instead, it prioritizes reaching :setting:`CONCURRENT_REQUESTS` and :ref:`scheduling <topics-scheduler>` start requests. +.. + The request send order when all start requests and callback requests have + the same priority is rather unintuitive: + + 1. First, the first CONCURRENT_REQUESTS start requests are sent in order. + + Awaiting slow operations in Spider.start() can lower that. + + 2. Then, assuming an even domain distribution in start requests (i.e. + ABCABC, not AABBCC), the last N start requests are sent in reverse + order, where N is: + + min(CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN * domain_count) + + 3. Finally, the remaining start requests are also sent in reverse order, + but only when there are not enough pending requests yielded from + callbacks to reach the configured concurrency. + + The reverse order is because the scheduler uses a LIFO queue by default + (SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few + requests is unnaffected because they are sent as soon as they are + scheduled. The last start requests sent before callback requests are those + that can be sent before the first callback requests are scheduled. + + We do not document this behavior, so that we may change it in the future + without breaking the contract. + +.. _start-requests-order: + Forcing a start request order ----------------------------- @@ -405,6 +434,8 @@ To force a specific **request order**, override the You can also :ref:`customize the scheduler <topics-scheduler>` if you need more control over request prioritization. +.. _start-requests-lazy: + Delaying start request iteration -------------------------------- diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a43f55137..0dd9954cb 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: import argparse - from collections.abc import AsyncGenerator, AsyncIterable, Coroutine, Iterable + from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable from twisted.python.failure import Failure @@ -258,7 +258,7 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - async def start(spider: Spider) -> AsyncIterable[Any]: + async def start(spider: Spider) -> AsyncIterator[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4ef996464..1bbf8921f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -241,6 +241,11 @@ class ExecutionEngine: self._spider_idle() def needs_backout(self) -> bool: + """Returns ``True`` if no more requests can be sent at the moment, or + ``False`` otherwise. + + See :ref:`start-requests-lazy` for an example. + """ assert self._slot is not None # typing assert self.scraper.slot is not None # typing return ( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 496adb500..85dd9b559 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,7 +5,7 @@ from __future__ import annotations import logging from collections import deque -from collections.abc import AsyncIterable, Iterator +from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -170,7 +170,7 @@ class Scraper: raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( + dfd: Deferred[Iterable[Any] | AsyncIterator[Any]] = self._scrape2( result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) @@ -181,7 +181,7 @@ class Scraper: def _scrape2( self, result: Response | Failure, request: Request, spider: Spider - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]: """ Handle the different cases of request's result been a Response or a Failure """ @@ -197,7 +197,7 @@ class Scraper: def call_spider( self, result: Response | Failure, request: Request, spider: Spider - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]: dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: @@ -216,7 +216,7 @@ class Scraper: if request.errback: warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) - dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( + dfd2: Deferred[Iterable[Any] | AsyncIterator[Any]] = dfd.addCallback( iterate_spider_output ) return dfd2 @@ -253,16 +253,16 @@ class Scraper: def handle_spider_output( self, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], request: Request, response: Response, spider: Spider, ) -> _HandleOutputDeferred: if not result: return defer_succeed(None) - it: Iterable[_T] | AsyncIterable[_T] + it: Iterable[_T] | AsyncIterator[_T] dfd: Deferred[_ParallelResult] - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): it = aiter_errback( result, self.handle_spider_error, request, response, spider ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index f8946b221..ec125bb3e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable +from collections.abc import AsyncIterator, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -41,12 +41,12 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterator[_T]] ] def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterable)) + return isinstance(o, (Iterable, AsyncIterator)) class SpiderMiddlewareManager(MiddlewareManager): @@ -136,7 +136,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Iterable[_T] | AsyncIterator[_T]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -157,10 +157,10 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Iterable[_T] | AsyncIterable[_T], + iterable: Iterable[_T] | AsyncIterator[_T], exception_processor_index: int, recover_to: MutableChain[_T] | MutableAsyncChain[_T], - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Iterable[_T] | AsyncIterator[_T]: def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable @@ -176,7 +176,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: + async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]: try: async for r in iterable: yield r @@ -192,7 +192,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) - if isinstance(iterable, AsyncIterable): + if isinstance(iterable, AsyncIterator): return process_async(iterable) return process_sync(iterable) @@ -251,13 +251,13 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], start_index: int = 0, ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. @@ -284,7 +284,7 @@ class SpiderMiddlewareManager(MiddlewareManager): need_downgrade = True try: if need_upgrade: - # Iterable -> AsyncIterable + # Iterable -> AsyncIterator result = as_async_generator(result) elif need_downgrade: logger.warning( @@ -294,10 +294,10 @@ class SpiderMiddlewareManager(MiddlewareManager): f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) - assert isinstance(result, AsyncIterable) - # AsyncIterable -> Iterable + assert isinstance(result, AsyncIterator) + # AsyncIterator -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = yield deferred_from_coro( collect_asyncgen(recovered) ) @@ -330,7 +330,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"{type(result)}" ) raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) if last_result_is_async: return MutableAsyncChain(result, recovered) @@ -340,23 +340,23 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: recovered: MutableChain[_T] | MutableAsyncChain[_T] - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( cast( - "Deferred[Iterable[_T] | AsyncIterable[_T]]", + "Deferred[Iterable[_T] | AsyncIterator[_T]]", self._process_spider_output(response, spider, result), ) ) - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = await collect_asyncgen(recovered) recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) @@ -369,7 +369,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: async def process_callback_output( - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) @@ -378,7 +378,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred[Iterable[_T] | AsyncIterable[_T]] = mustbe_deferred( + dfd: Deferred[Iterable[_T] | AsyncIterator[_T]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 5bdc1b690..f8c50b5e3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -76,7 +76,10 @@ class SignalManager: _signal.disconnect_all(signal, **kwargs) async def wait_for(self, signal): - """Await the next *signal*.""" + """Await the next *signal*. + + See :ref:`start-requests-lazy` for an example. + """ d = Deferred() def handle(): diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 3164c1c03..5760411c0 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy.http import Request, Response if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -54,8 +54,8 @@ class DepthMiddleware: return (r for r in result if self._filter(r, response, spider)) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: self._init_depth(response, spider) async for r in result: if self._filter(r, response, spider): diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 646beb911..060be1975 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -23,7 +23,7 @@ warnings.warn( ) if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,8 +52,8 @@ class OffsiteMiddleware: return (r for r in result if self._filter(r, spider)) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: async for r in result: if self._filter(r, spider): yield r diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a3a1e5b92..c824579b0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -19,7 +19,7 @@ from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -376,8 +376,8 @@ class RefererMiddleware: return (self._set_referer(r, response) for r in result) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: async for r in result: yield self._set_referer(r, response) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a1cd1bb7c..9259fea3c 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,7 +14,7 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -57,8 +57,8 @@ class UrlLengthMiddleware: return (r for r in result if self._filter(r, spider)) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: async for r in result: if self._filter(r, spider): yield r diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 087049425..171d8479c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from collections.abc import AsyncIterable, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast from twisted.python.failure import Failure @@ -156,10 +156,10 @@ class CrawlSpider(Spider): callback: CallbackT | None, cb_kwargs: dict[str, Any], follow: bool = True, - ) -> AsyncIterable[Any]: + ) -> AsyncIterator[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, AsyncIterable): + if isinstance(cb_res, AsyncIterator): cb_res = await collect_asyncgen(cb_res) elif isinstance(cb_res, Awaitable): cb_res = await cb_res diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 237bd8331..6d96a41f5 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,20 +1,20 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from collections.abc import AsyncGenerator, AsyncIterator, Iterable from typing import TypeVar _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: +async def collect_asyncgen(result: AsyncIterator[_T]) -> list[_T]: return [x async for x in result] async def as_async_generator( - it: Iterable[_T] | AsyncIterable[_T], + it: Iterable[_T] | AsyncIterator[_T], ) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" - if isinstance(it, AsyncIterable): + if isinstance(it, AsyncIterator): async for r in it: yield r else: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 42ad28d8d..c39837716 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -22,7 +22,7 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: - from collections.abc import AsyncIterable, AsyncIterator, Callable + from collections.abc import AsyncIterator, Callable from twisted.python.failure import Failure @@ -177,7 +177,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, @@ -234,7 +234,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( - async_iterable: AsyncIterable[_T], + async_iterable: AsyncIterator[_T], count: int, callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *args: _P.args, @@ -332,11 +332,11 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], errback: Callable[Concatenate[Failure, _P], Any], *a: _P.args, **kw: _P.kwargs, -) -> AsyncIterable[_T]: +) -> AsyncIterator[_T]: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 2e6869779..7e7b83c59 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import re import sys import warnings import weakref -from collections.abc import AsyncIterable, Iterable, Mapping +from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, TypeVar, overload @@ -19,8 +19,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator + from collections.abc import Callable, Iterator from re import Pattern + from typing import Self # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec @@ -369,25 +370,25 @@ class MutableChain(Iterable[_T]): async def _async_chain( - *iterables: Iterable[_T] | AsyncIterable[_T], + *iterables: Iterable[_T] | AsyncIterator[_T], ) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable[_T]): +class MutableAsyncChain(AsyncIterator[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Iterable[_T] | AsyncIterable[_T]): + def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]): self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Iterable[_T] | AsyncIterable[_T]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterator[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator[_T]: + def __aiter__(self) -> Self: return self async def __anext__(self) -> _T: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 8ef62d018..c1bca8f9a 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -41,7 +41,7 @@ class MainTestCase(TestCase): self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) - # During this time, the reactor reports having requests but + # During this time, the scheduler reports having requests but # returns None. await sleep(seconds) @@ -81,33 +81,6 @@ class MainTestCase(TestCase): class RequestSendOrderTestCase(TestCase): - """Test the intrincacies of request send order when all start requests and - callback requests have the same priority. - - It is a very unintuitive behavior, documented as “undefined” so that we may - change it in the future without breaking the contract: - - 1. First, the first CONCURRENT_REQUESTS start requests are sent in order. - - Awaiting slow operations in Spider.start() can lower that. - - 2. Then, assuming an even domain distribution in start requests (i.e. - ABCABC, not AABBCC), the last N start requests are sent in reverse - order, where N is: - - min(CONCURRENT_REQUESTS, CONCURRENT_REQUESTS_PER_DOMAIN * domain_count) - - 3. Finally, the remaining start requests are also sent in reverse order, - but only when there are not enough pending requests yielded from - callbacks to reach the configured concurrency. - - The reverse order is because the scheduler uses a LIFO queue by default - (SCHEDULER_MEMORY_QUEUE, SCHEDULER_DISK_QUEUE). The order of the first few - requests is unnaffected because they are sent as soon as they are - scheduled. The last start requests sent before callback requests are those - that can be sent before the first callback requests are scheduled. - """ - seconds = 0.1 # increase if flaky @classmethod @@ -238,9 +211,8 @@ class RequestSendOrderTestCase(TestCase): cb_nums=cb_nums, settings={ "CONCURRENT_REQUESTS": 1, - # Without the lazy approach, using the FIFO queue would - # yield a different result, with start requests not being - # sorted. + # Without the lazy approach, a FIFO queue would yield the + # start requests in a different order. "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.FifoMemoryQueue", }, response_seconds=response_seconds, diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 006701220..b5eaa93f5 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -6,17 +6,11 @@ from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from scrapy import Spider, signals -from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler -# These are the minimum seconds necessary to wait to reproduce the issue that -# has been solved by catching the RuntimeError exception in the -# ExecutionEngine._next_request() method. A lower value makes these tests pass -# even if we remove that exception handling, but they start failing with this -# much delay. -ASYNC_GEN_ERROR_MINIMUM_SECONDS = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01 +SLEEP_SECONDS = 0.1 ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} @@ -138,7 +132,7 @@ class MainTestCase(TestCase): @deferred_f_from_coro_f async def test_asyncio_delayed(self): async def start(spider): - await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) + await sleep(SLEEP_SECONDS) yield ITEM_A await self._test_start(start, [ITEM_A]) @@ -146,9 +140,7 @@ class MainTestCase(TestCase): @deferred_f_from_coro_f async def test_twisted_delayed(self): async def start(spider): - await maybe_deferred_to_future( - twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) - ) + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) yield ITEM_A await self._test_start(start, [ITEM_A]) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9728d0d81..9cd7b44ca 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -7,7 +7,6 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -16,7 +15,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.test import get_crawler @@ -201,7 +204,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware: yield {"foo": 3} -class ProcessSpiderExceptionAsyncIterableMiddleware: +class ProcessSpiderExceptionAsyncIteratorMiddleware: async def process_spider_exception(self, response, exception, spider): yield {"foo": 1} d = defer.Deferred() @@ -332,8 +335,7 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessStartSimpleMiddleware - @inlineCallbacks - def _get_processed_start(self, *mw_classes): + async def _get_processed_start(self, *mw_classes): class TestSpider(Spider): name = "test" @@ -348,15 +350,14 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): ) self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - results = yield self.mwman.process_start(self.spider) - return results + return await maybe_deferred_to_future(self.mwman.process_start(self.spider)) - @inlineCallbacks - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw""" - start = yield self._get_processed_start(self.MW_SIMPLE) + start = await self._get_processed_start(self.MW_SIMPLE) assert isasyncgen(start) - start_list = yield deferred_from_coro(collect_asyncgen(start)) + start_list = await collect_asyncgen(start) assert len(start_list) == self.RESULT_COUNT assert isinstance(start_list[0], self.ITEM_TYPE) @@ -516,7 +517,7 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware - MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware def _scrape_func(self, *args, **kwargs): 1 / 0 diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 1913b4251..e2669a716 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -9,7 +9,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from .test_spider_start import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep +from .test_spider_start import SLEEP_SECONDS, twisted_sleep ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} @@ -19,7 +19,7 @@ ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: async def process_start(self, start): - await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) + await sleep(SLEEP_SECONDS) async for item_or_request in start: yield item_or_request @@ -32,7 +32,7 @@ class NoOpSpiderMiddleware: class TwistedSleepSpiderMiddleware: async def process_start(self, start): - await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)) + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) async for item_or_request in start: yield item_or_request From c67edb78b95bd21813a8fc6a26b06514c1161c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 15:04:34 +0100 Subject: [PATCH 39/45] Deprecate calls to Spider.start_requests --- scrapy/spiders/__init__.py | 20 ++++++++++++++++++-- scrapy/spiders/init.py | 8 ++++++-- scrapy/utils/python.py | 5 ++--- tests/test_spider_start.py | 36 +++++++++++++++++++++++++++++++----- tox.ini | 2 +- 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index acd16ee41..0a1d85ae6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,9 +7,11 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider @@ -124,10 +126,24 @@ class Spider(object_ref): .. seealso:: :ref:`start-requests` """ - for item_or_request in self.start_requests(): - yield item_or_request + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Any]: + warnings.warn( + ( + "The Spider.start_requests() method is deprecated, use " + "Spider.start() instead. If you are calling " + "super().start_requests() from a Spider.start() override, " + "iterate super().start() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ca5f08e98..e5548b9fa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -30,8 +30,12 @@ class InitSpider(Spider): ) async def start(self) -> AsyncIterator[Any]: - for item_or_request in self.start_requests(): - yield item_or_request + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7e7b83c59..1d9e34f35 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -21,10 +21,9 @@ from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: from collections.abc import Callable, Iterator from re import Pattern - from typing import Self - # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Concatenate, typing.ParamSpec and typing.Self require Python 3.10 + from typing_extensions import Concatenate, ParamSpec, Self _P = ParamSpec("_P") diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index b5eaa93f5..02f8770e9 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,3 +1,4 @@ +import warnings from asyncio import sleep import pytest @@ -47,7 +48,9 @@ class MainTestCase(TestCase): async def parse(self, response): yield ITEM_A - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_start(self): @@ -57,7 +60,9 @@ class MainTestCase(TestCase): async def start(self): yield ITEM_A - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_start_subclass(self): @@ -68,7 +73,9 @@ class MainTestCase(TestCase): class TestSpider(BaseSpider): name = "test" - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_deprecated(self): @@ -105,7 +112,9 @@ class MainTestCase(TestCase): def start_requests(self): yield ITEM_B - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f async def test_universal_subclass(self): @@ -119,7 +128,24 @@ class MainTestCase(TestCase): class TestSpider(BaseSpider): name = "test" - await self._test_spider(TestSpider, [ITEM_A]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_deprecated_super(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + for item_or_request in super().start_requests(): + yield item_or_request + + with pytest.warns( + ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead" + ) as messages: + await self._test_spider(TestSpider, []) + assert messages[0].filename.endswith("test_spider_start.py") async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): diff --git a/tox.ini b/tox.ini index 59572442d..089f48f86 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] -basepython = python3 +basepython = python3.9 deps = mypy==1.14.0 typing-extensions==4.12.2 From 8057f06b3991011d1f8d6fca7a5f8444ff0c8b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 15:16:02 +0100 Subject: [PATCH 40/45] Increase coverage --- tests/test_spidermiddleware_process_start.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index e2669a716..9265b9d35 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -56,6 +56,10 @@ class ModernWrapSpider(Spider): yield ITEM_B +class ModernWrapSpiderSubclass(ModernWrapSpider): + name = "test" + + class UniversalWrapSpider(Spider): name = "test" @@ -175,6 +179,21 @@ class MainTestCase(TestCase): ): await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider_subclass(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, + match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders", + ), + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass + ) + @deferred_f_from_coro_f async def test_deprecated_mw_universal_spider(self): with pytest.warns( From e360a4cc98751c14d3fde6258154d1500c360c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 15:24:57 +0100 Subject: [PATCH 41/45] Increase coverage --- tests/test_spider.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index ee159a134..36abdd1c5 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -26,6 +26,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -774,6 +775,24 @@ Sitemap: /sitemap-relative-url.xml ), ) + @deferred_f_from_coro_f + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + with warnings.catch_warnings(): + warnings.simplefilter("error") + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap + class TestDeprecation: def test_crawl_spider(self): From 40adc6a3567eb405a58bb5de94999b1da476b7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 15:39:37 +0100 Subject: [PATCH 42/45] Increase coverage --- tests/test_spider.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 36abdd1c5..b4aa649a3 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -26,7 +26,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -146,6 +146,22 @@ class TestSpider(unittest.TestCase): class TestInitSpider(TestSpider): spider_class = InitSpider + @deferred_f_from_coro_f + async def test_start_urls(self): + responses = [] + + class TestSpider(self.spider_class): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + responses.append(response) + + crawler = get_crawler(TestSpider) + await maybe_deferred_to_future(crawler.crawl()) + assert len(responses) == 1 + assert responses[0].url == "data:," + class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider From 2a148d07b0169f4239bd3949d32bf583050f5876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 15:51:08 +0100 Subject: [PATCH 43/45] Increase coverage --- tests/test_engine_loop.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c1bca8f9a..0c478f4e3 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -1,5 +1,7 @@ from collections import deque +from logging import ERROR +from testfixtures import LogCapture from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -79,6 +81,37 @@ class MainTestCase(TestCase): expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + @deferred_f_from_coro_f + async def test_close_during_start_iteration(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + assert self.crawler.engine is not None + await maybe_deferred_to_future(self.crawler.engine.close()) + yield Request("data:,a") + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + + with LogCapture(level=ERROR) as log: + await maybe_deferred_to_future(crawler.crawl()) + + assert not log.records, f"{log.records=}" + finish_reason = crawler.stats.get_value("finish_reason") + assert finish_reason == "shutdown", f"{finish_reason=}" + expected_urls = [] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + class RequestSendOrderTestCase(TestCase): seconds = 0.1 # increase if flaky From 7d159b2140dc60f84d42f7bfca3556fcb89755bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 15:56:34 +0100 Subject: [PATCH 44/45] Privatize engine.start_request_processing() --- scrapy/core/engine.py | 9 ++------- scrapy/shell.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1bbf8921f..d5e3f4064 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -102,7 +102,6 @@ class ExecutionEngine: ) self.start_time: float | None = None self._start: AsyncIterator[Any] | None = None - self._started_request_processing = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -135,7 +134,7 @@ class ExecutionEngine: ) self.running = True if _start_request_processing: - self.start_request_processing() + self._start_request_processing() self._closewait: Deferred[None] = Deferred() await maybe_deferred_to_future(self._closewait) @@ -207,13 +206,9 @@ class ExecutionEngine: self._slot.nextcall.schedule() @deferred_f_from_coro_f - async def start_request_processing(self) -> None: + async def _start_request_processing(self) -> None: """Starts consuming Spider.start() output and sending scheduled requests.""" - if self._started_request_processing: - raise RuntimeError("Request processing already started") - self._started_request_processing = True - # Starts the processing of scheduled requests, as well as a periodic # call to that processing method for scenarios where the scheduler # reports having pending requests but returns none. diff --git a/scrapy/shell.py b/scrapy/shell.py index 14bc31137..bb39eccc3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -128,7 +128,7 @@ class Shell: await maybe_deferred_to_future( self.crawler.engine.open_spider(spider, close_if_idle=False) ) - self.crawler.engine.start_request_processing() + self.crawler.engine._start_request_processing() self.spider = spider def fetch( From 8e3d211ad7a61317fe9945f47da1ae0e283d3e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 27 Mar 2025 16:37:19 +0100 Subject: [PATCH 45/45] Fix test_close_during_start_iteration on default-reactor --- scrapy/core/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d5e3f4064..09cb30daf 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -133,9 +133,9 @@ class ExecutionEngine: self.signals.send_catch_log_deferred(signal=signals.engine_started) ) self.running = True + self._closewait: Deferred[None] = Deferred() if _start_request_processing: self._start_request_processing() - self._closewait: Deferred[None] = Deferred() await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: