diff --git a/docs/contributing.rst b/docs/contributing.rst index bb197b428..34e92d8f0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -228,8 +228,8 @@ with a name of the branch you want to create locally). See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. -E.g. For bug #411: "Scrapy hangs if an exception raises in yield_seeds" prefer -"Fix hanging when exception occurs in yield_seeds (#411)" instead of "Fix for +E.g. For bug #411: "Scrapy hangs if an exception raises in start" prefer +"Fix hanging when exception occurs in start (#411)" instead of "Fix for #411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8f74f76af..71b420be6 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - async def yield_seeds(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,7 +116,7 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.yield_seeds`: must be an asynchronous generator that +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that yields requests (and, optionally, items) for the spider to start crawling. Subsequent requests will be generated successively from these initial requests. @@ -165,20 +165,20 @@ What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Scrapy sends the first :class:`scrapy.Request ` objects yielded -by the :meth:`~scrapy.Spider.yield_seeds` spider method. Upon receiving a +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a response for each one, Scrapy calls the callback method associated with the request (in this case, the ``parse`` method) with a :class:`~scrapy.http.Response` object. -A shortcut to the ``yield_seeds`` method +A shortcut to the ``start`` method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.yield_seeds` method that yields +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields :class:`~scrapy.Request` objects from URLs, you can define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation of -:meth:`~scrapy.Spider.yield_seeds` to create the initial requests for your +:meth:`~scrapy.Spider.start` to create the initial requests for your spider. .. code-block:: python @@ -795,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - async def yield_seeds(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 93e003399..de7506e42 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,20 +10,20 @@ Scrapy VERSION (unreleased) Highlights: -- Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.yield_seeds` +- Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.start` (async) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - In ``scrapy.core.spidermw.SpiderMiddlewareManager``, - ``process_start_requests()`` has been replaced by ``process_seeds()``. + ``process_start_requests()`` has been replaced by ``process_start()``. - In ``scrapy.core.engine.ExecutionEngine``: - The second parameter of ``open_spider()``, ``start_requests``, has been removed. The starting requests are determined by the ``spider`` - parameter instead (see :meth:`~scrapy.Spider.yield_seeds`). + parameter instead (see :meth:`~scrapy.Spider.start`). - The ``slot`` attribute has been renamed to ``_slot`` and should not be used. @@ -37,7 +37,7 @@ Deprecations ~~~~~~~~~~~~ - The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, - use :meth:`~scrapy.Spider.yield_seeds` instead, or both to maintain support + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for lower Scrapy versions. (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, @@ -45,7 +45,7 @@ Deprecations - The ``process_start_requests()`` method of :ref:`spider middlewares ` is deprecated, use - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` instead, or + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, or both to maintain support for lower Scrapy versions. (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, @@ -55,8 +55,8 @@ New features ~~~~~~~~~~~~ - You can now yield the start requests and items of a spider from the - :meth:`~scrapy.Spider.yield_seeds` spider method and from the - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` spider + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider middleware method, both asynchronous generators. This makes it possible to use asynchronous code to generate those start @@ -68,7 +68,7 @@ New features Bug fixes ~~~~~~~~~ -- Yielding a start item (i.e. from :meth:`~scrapy.Spider.yield_seeds` or an +- Yielding a start item (i.e. from :meth:`~scrapy.Spider.start` or an equivalent) no longer delays the next iteration of starting requests and items by up to 5 seconds. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index bed13f483..2a7a0c7c7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -18,7 +18,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- The :meth:`~scrapy.spiders.Spider.yield_seeds` spider method. +- The :meth:`~scrapy.spiders.Spider.start` spider method. .. versionadded: VERSION @@ -54,7 +54,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 -- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` method +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of :ref:`spider middlewares `. .. versionadded:: VERSION @@ -157,7 +157,7 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in - :meth:`~scrapy.spiders.Spider.yield_seeds`, callbacks, pipelines and + :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a81832023..2032fc5cf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -353,7 +353,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - async def yield_seeds(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5bc1ccc7d..67de4f5f1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1950,7 +1950,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - async def yield_seeds(self): + async def start(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1979,7 +1979,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - async def yield_seeds(self): + async def start(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 782ac1e26..091a4c000 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -160,7 +160,7 @@ item_scraped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.yield_seeds`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped @@ -181,7 +181,7 @@ item_dropped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.yield_seeds`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a @@ -205,7 +205,7 @@ item_error :param response: the response being processed when the exception was raised, or ``None`` if it was yielded from - :meth:`~scrapy.Spider.yield_seeds`. + :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 55330da9f..4632644e6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,21 +70,21 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_seeds(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] + .. method:: process_start(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] :async: - Iterate over the output of :meth:`~scrapy.Spider.yield_seeds` or that - of the :meth:`process_seeds` method of an earlier spider middleware, + Iterate over the output of :meth:`~scrapy.Spider.start` or that + of the :meth:`process_start` method of an earlier spider middleware, overriding it. For example: .. code-block:: python - async def process_seeds(self, seeds): + async def process_start(self, seeds): async for seed in seeds: yield seed You may yield :class:`~scrapy.Request` or :ref:`item ` - objects, same as :meth:`~scrapy.Spider.yield_seeds`, from *seeds* or + objects, same as :meth:`~scrapy.Spider.start`, from *seeds* or not. To write spider middlewares that work on Scrapy versions lower than diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 88eed477a..a05a4b14d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,7 +17,7 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by iterating the - :meth:`~scrapy.Spider.yield_seeds` method, which by default yields a + :meth:`~scrapy.Spider.start` method, which by default yields a :class:`~scrapy.Request` object for each URL in the :attr:`~scrapy.Spider.start_urls` spider attribute, with the :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` @@ -137,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`yield_seeds` method, handlers of the + :meth:`start` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -189,7 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. automethod:: yield_seeds + .. automethod:: start .. method:: parse(response) @@ -261,7 +261,7 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.yield_seeds` directly; +Instead of :attr:`~.start_urls` you can use :meth:`~.start` directly; to give data more structure you can use :class:`~scrapy.Item` objects: .. skip: next @@ -275,7 +275,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - async def yield_seeds(self): + async def start(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -329,7 +329,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - async def yield_seeds(self): + async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") If you are :ref:`running Scrapy from a script `, you can @@ -893,8 +893,8 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - async def yield_seeds(self): - async for seed in super().yield_seeds(): + async def start(self): + async for seed in super().start(): yield seed for url in self.other_urls: yield Request(url, self.parse_other) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index df1bf8767..f95c6010f 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,7 +34,7 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) - async def yield_seeds(self): + async def start(self): for seed in self.start_requests(): yield seed diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 5ed09c77c..4cd5d640a 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -59,7 +59,7 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" yield scrapy.Request(url, dont_filter=True) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 24dfc0106..59c5ab9b5 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -80,7 +80,7 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - async def yield_seeds(self): + async def start(self): for request in conman.from_spider(self, self._result): yield request @@ -89,7 +89,7 @@ class Command(ScrapyCommand): spidercls = spider_loader.load(spidername) spidercls._result = result # type: ignore[assignment,attr-defined,method-assign,return-value] - spidercls.yield_seeds = yield_seeds # type: ignore[assignment,method-assign,return-value] + spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d1b0974ec..77c318958 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -90,11 +90,11 @@ class Command(ScrapyCommand): else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - async def yield_seeds(self): + async def start(self): yield self._request spidercls._request = request # type: ignore[assignment,attr-defined] - spidercls.yield_seeds = yield_seeds # type: ignore[method-assign,attr-defined] + spidercls.start = start # type: ignore[method-assign,attr-defined] self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 810824c7e..a43f55137 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -258,11 +258,11 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - async def yield_seeds(spider: Spider) -> AsyncIterable[Any]: + async def start(spider: Spider) -> AsyncIterable[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.yield_seeds = yield_seeds # type: ignore[assignment,method-assign] + self.spidercls.start = start # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index a37dc6b13..23cddf481 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -380,7 +380,7 @@ class ExecutionEngine: logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._start_next_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - self._seeds = yield self.scraper.spidermw.process_seeds(spider) + self._seeds = yield self.scraper.spidermw.process_start(spider) self._slot = _Slot(close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open") and (d := scheduler.open(spider)): diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 287b15c4b..3c1411a18 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -56,7 +56,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): The original sources of said requests are: - * Spider: ``yield_seeds`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``start`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0affa4534..9a292cae9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -67,7 +67,7 @@ class SpiderMiddlewareManager(MiddlewareManager): middleware for middleware in middlewares if hasattr(middleware, "process_start_requests") - and not hasattr(middleware, "process_seeds") + and not hasattr(middleware, "process_start") ] self._use_start_requests = bool(deprecated_middlewares) if self._use_start_requests: @@ -80,16 +80,16 @@ class SpiderMiddlewareManager(MiddlewareManager): f"through their parent classes, define the deprecated " f"process_start_requests() method: " f"{deprecated_middleware_list}. process_start_requests() has " - f"been deprecated in favor of a new method, process_seeds(), " + f"been deprecated in favor of a new method, process_start(), " f"to support asynchronous code execution. " f"process_start_requests() will stop being called in a future " f"version of Scrapy. If you use Scrapy VERSION or higher " f"only, replace process_start_requests() with " - f"process_seeds(); note that process_seeds() is a coroutine " + f"process_start(); note that process_start() is a coroutine " f"(async def). If you need to maintain compatibility with " f"lower Scrapy versions, when defining " f"process_start_requests() in a spider middleware class, " - f"define process_seeds() as well. See the release notes of " + f"define process_start() as well. See the release notes of " f"Scrapy VERSION for details: " f"https://docs.scrapy.org/en/VERSION/news.html", ScrapyDeprecationWarning, @@ -104,8 +104,8 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods["process_start_requests"].appendleft( mw.process_start_requests ) - elif hasattr(mw, "process_seeds"): - self.methods["process_seeds"].appendleft(mw.process_seeds) + elif hasattr(mw, "process_start"): + self.methods["process_start"].appendleft(mw.process_start) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) process_spider_exception = getattr(mw, "process_spider_exception", None) @@ -369,7 +369,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return dfd2 @inlineCallbacks - def process_seeds( + def process_start( self, spider: Spider ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]: self._check_deprecated_start_requests_use(spider) @@ -381,12 +381,12 @@ class SpiderMiddlewareManager(MiddlewareManager): seeds = as_async_generator(sync_seeds) else: seeds = yield self._iter_seeds(spider) - seeds = yield self._process_chain("process_seeds", seeds) + seeds = yield self._process_chain("process_start", seeds) return seeds def _check_deprecated_start_requests_use(self, spider: Spider): start_requests_cls = None - yield_seeds_cls = None + start_cls = None spidercls = spider.__class__ mro = spidercls.__mro__ @@ -394,19 +394,19 @@ class SpiderMiddlewareManager(MiddlewareManager): cls_dict = cls.__dict__ if start_requests_cls is None and "start_requests" in cls_dict: start_requests_cls = cls - if yield_seeds_cls is None and "yield_seeds" in cls_dict: - yield_seeds_cls = cls - if start_requests_cls is not None and yield_seeds_cls is not None: + if start_cls is None and "start" in cls_dict: + start_cls = cls + if start_requests_cls is not None and start_cls is not None: break - # Spider defines both, start_requests and yield_seeds. + # Spider defines both, start_requests and start. assert start_requests_cls is not None - assert yield_seeds_cls is not None + assert start_cls is not None if ( start_requests_cls is not Spider - and yield_seeds_cls is not start_requests_cls - and mro.index(start_requests_cls) < mro.index(yield_seeds_cls) + and start_cls is not start_requests_cls + and mro.index(start_requests_cls) < mro.index(start_cls) ): src = global_object_name(start_requests_cls) if start_requests_cls is not spidercls: @@ -414,15 +414,15 @@ class SpiderMiddlewareManager(MiddlewareManager): warn( f"{src} defines the deprecated start_requests() method. " f"start_requests() has been deprecated in favor of a new " - f"method, yield_seeds(), to support asynchronous code " + f"method, start(), to support asynchronous code " f"execution. start_requests() will stop being called in a " f"future version of Scrapy. If you use Scrapy VERSION or " - f"higher only, replace start_requests() with yield_seeds(); " - f"note that yield_seeds() is a coroutine (async def). If you " + f"higher only, replace start_requests() with start(); " + f"note that start() is a coroutine (async def). If you " f"need to maintain compatibility with lower Scrapy versions, " f"when overriding start_requests() in a spider class, " - f"override yield_seeds() as well; you can use super() to " - f"reuse the inherited yield_seeds() implementation without " + f"override start() as well; you can use super() to " + f"reuse the inherited start() implementation without " f"copy-pasting. See the release notes of Scrapy VERSION for " f"details: https://docs.scrapy.org/en/VERSION/news.html", ScrapyDeprecationWarning, @@ -430,33 +430,33 @@ class SpiderMiddlewareManager(MiddlewareManager): if ( self._use_start_requests - and yield_seeds_cls is not Spider - and start_requests_cls is not yield_seeds_cls - and mro.index(yield_seeds_cls) < mro.index(start_requests_cls) + and start_cls is not Spider + and start_requests_cls is not start_cls + and mro.index(start_cls) < mro.index(start_requests_cls) ): - src = global_object_name(yield_seeds_cls) - if yield_seeds_cls is not spidercls: + src = global_object_name(start_cls) + if start_cls is not spidercls: src += f" (inherited by {global_object_name(spidercls)})" raise ValueError( f"{src} does not define the deprecated start_requests() " f"method. However, one or more of your enabled spider " f"middlewares (reported in an earlier deprecation warning) " f"define the process_start_requests() method, and not the " - f"process_seeds() method, making them only compatible with " + f"process_start() method, making them only compatible with " f"(deprecated) spiders that define the start_requests() " f"method. To solve this issue, disable the offending spider " f"middlewares, upgrade them as described in that earlier " f"deprecation warning, or make your spider compatible with " f"deprecated spider middlewares (and earlier Scrapy versions) " f"by defining a sync start_requests() method that works " - f"similarly to its existing yield_seeds() method. See the " + f"similarly to its existing start() method. See the " f"release notes of Scrapy VERSION for details: " f"https://docs.scrapy.org/en/VERSION/news.html" ) @staticmethod def _iter_seeds(spider: Spider): - fn = spider.yield_seeds + fn = spider.start if isasyncgenfunction(fn): return fn().__aiter__() assert iscoroutinefunction(fn) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0d2b3aa99..d72c72ecd 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -191,7 +191,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.yield_seeds`. + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 6315b7adc..4f08918ae 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -98,7 +98,7 @@ class LogFormatter: """Logs a message when an item is scraped by a spider.""" src: Any if response is None: - src = f"{global_object_name(spider.__class__)}.yield_seeds" + src = f"{global_object_name(spider.__class__)}.start" elif isinstance(response, Failure): src = response.getErrorMessage() else: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 5f2ac1a26..eb88f62a4 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: class Spider(object_ref): """Base class that any spider must subclass. - It provides a default :meth:`yield_seeds` implementation that sends + It provides a default :meth:`start` implementation that sends requests based on the :attr:`start_urls` class attribute and calls the :meth:`parse` method for each response. """ @@ -39,7 +39,7 @@ class Spider(object_ref): name: str custom_settings: dict[_SettingsKeyT, Any] | None = None - #: Seed URLs. See :meth:`yield_seeds`. + #: Seed URLs. See :meth:`start`. start_urls: list[str] def __init__(self, name: str | None = None, **kwargs: Any): @@ -78,7 +78,7 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. .. versionadded:: VERSION @@ -93,7 +93,7 @@ class Spider(object_ref): class MySpider(Spider): name = "myspider" - async def yield_seeds(self): + async def start(self): yield Request("https://toscrape.com/") The default implementation reads URLs from :attr:`start_urls` and @@ -102,7 +102,7 @@ class Spider(object_ref): .. code-block:: python - async def yield_seeds(self): + async def start(self): for url in self.start_urls: yield Request(url, dont_filter=True) @@ -110,7 +110,7 @@ class Spider(object_ref): .. code-block:: python - async def yield_seeds(self): + async def start(self): yield {"foo": "bar"} To write spiders that work on Scrapy versions lower than VERSION, diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index c1d38854b..ee0058880 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -29,7 +29,7 @@ class InitSpider(Spider): stacklevel=2, ) - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: for seed in self.start_requests(): yield seed diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index bc004f663..c01b54b4a 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -53,7 +53,7 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - async def yield_seeds(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterable[Any]: for seed in self.start_requests(): yield seed diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8b8ab927d..59a3dca21 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,8 +43,8 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - async def process_seeds(self, seeds): - # Called with the seeds from the spider yield_seeds() method or with + async def process_start(self, seeds): + # Called with the seeds from the spider start() method or with # the output of the maching method of an earlier spider middleware. async for seed in seeds: yield seed diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index 6076211ec..c8a3d0a5b 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,7 +13,7 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - async def yield_seeds(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") return yield diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 13c6fff85..bd78a0de7 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 950bdb006..6bb6fb3c6 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -12,7 +12,7 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index a80a1180a..f3dab12fe 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -38,7 +38,7 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 59feced94..d8c467f40 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -15,7 +15,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index f297fd0d7..e7d3ca9cc 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -16,7 +16,7 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 26343da6a..53d427061 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - async def yield_seeds(self): + async def start(self): yield scrapy.Request(self.url) def parse(self, response): diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 65f5e033f..0058896b5 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index a221764d2..8f59c035c 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -8,7 +8,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index a0aff999b..9901dd634 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -8,7 +8,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 6ac1043b5..53941568a 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -10,7 +10,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index f7352af57..5739d77ae 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,7 +17,7 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index 1071e453d..c488f7526 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,7 +9,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 3773092b0..9e4ad70d9 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index bdc217fde..6c0102241 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -10,7 +10,7 @@ class NoRequestsSpider(Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 892fab731..5e2184afb 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -32,7 +32,7 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - async def yield_seeds(self): + async def start(self): yield Request(self.url) def parse(self, response): diff --git a/tests/spiders.py b/tests/spiders.py index 5075a795b..d7fffd001 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -68,7 +68,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - async def yield_seeds(self): + async def start(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -105,7 +105,7 @@ class LogSpider(MetaSpider): class SlowSpider(DelaySpider): name = "slow" - async def yield_seeds(self): + async def start(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -255,7 +255,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - async def yield_seeds(self): + async def start(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -327,7 +327,7 @@ class BrokenYieldSeedsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - async def yield_seeds(self): + async def start(self): if self.fail_before_yield: 1 / 0 @@ -346,12 +346,12 @@ class BrokenYieldSeedsSpider(FollowAllSpider): class YieldSeedsItemSpider(FollowAllSpider): - async def yield_seeds(self): + async def start(self): yield {"name": "test item"} class YieldSeedsGoodAndBadOutput(FollowAllSpider): - async def yield_seeds(self): + async def start(self): yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -363,7 +363,7 @@ class SingleRequestSpider(MetaSpider): callback_func = None errback_func = None - async def yield_seeds(self): + async def start(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -390,7 +390,7 @@ class DuplicateYieldSeedsSpider(MockServerSpider): distinct_urls = 2 dupe_factor = 3 - async def yield_seeds(self): + async def start(self): for i in range(self.distinct_urls): for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") @@ -415,7 +415,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - async def yield_seeds(self): + async def start(self): test_body = b""" Page title<title></head> @@ -469,7 +469,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - async def yield_seeds(self): + async def start(self): test_body = b""" <html> <head><title>Page title<title></head> @@ -514,7 +514,7 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - async def yield_seeds(self): + async def start(self): body = b"a" * self.full_response_length url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=body, errback=self.errback) @@ -543,7 +543,7 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status"), errback=self.errback) def parse(self, response): diff --git a/tests/test_commands.py b/tests/test_commands.py index 84ac8b125..7d399d474 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -670,7 +670,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug("It Works!") return yield @@ -681,7 +681,7 @@ import scrapy class BadSpider(scrapy.Spider): name = "bad" - async def yield_seeds(self): + async def start(self): raise Exception("oops!") """ @@ -772,9 +772,9 @@ class MySpider(scrapy.Spider): log = self.get_log("", name="myspider.txt") assert "Unable to load" in log - def test_yield_seeds_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.py") - assert "yield_seeds" in log + assert "start" in log assert "badspider.py" in log def test_asyncio_enabled_true(self): @@ -847,7 +847,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return yield @@ -864,7 +864,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) @@ -891,7 +891,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): return yield """ @@ -908,7 +908,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return yield @@ -988,7 +988,7 @@ class MySpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - async def yield_seeds(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") return yield @@ -1007,9 +1007,9 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): raise unittest.SkipTest("Windows required for .pyw files") return super().setUp() - def test_yield_seeds_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - assert "yield_seeds" in log + assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): @@ -1059,7 +1059,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('It works!') return yield @@ -1074,7 +1074,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return yield @@ -1091,7 +1091,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) @@ -1118,7 +1118,7 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - async def yield_seeds(self): + async def start(self): return yield """ diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d06e186cf..00bfe0ef9 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -511,7 +511,7 @@ class TestContractsManager(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - async def yield_seeds(self_): # pylint: disable=no-self-argument + async def start(self_): # pylint: disable=no-self-argument for seed in self.conman.from_spider(self_, self.results): yield seed diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 765d2c0f6..85d016e44 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -163,7 +163,7 @@ class TestCrawl(TestCase): self._assert_retried(log) @defer.inlineCallbacks - def test_yield_seeds_bug_before_yield(self): + def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenYieldSeedsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) @@ -174,7 +174,7 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_yield_seeds_bug_yielding(self): + def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenYieldSeedsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) @@ -185,7 +185,7 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_yield_seeds_items(self): + def test_start_items(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(YieldSeedsItemSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -193,7 +193,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 @defer.inlineCallbacks - def test_yield_seeds_unsupported_output(self): + def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item, and letting instead things fail when ItemAdapter is actually used on the corresponding @@ -205,7 +205,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 @defer.inlineCallbacks - def test_yield_seeds_laziness(self): + def test_start_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(BrokenYieldSeedsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) @@ -214,7 +214,7 @@ class TestCrawl(TestCase): ), crawler.spider.seedsseen @defer.inlineCallbacks - def test_yield_seeds_dupes(self): + def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateYieldSeedsSpider, settings) yield crawler.crawl( @@ -304,10 +304,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # test_yield_seeds doesn't set Referer header + # test_start doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to test_yield_seeds url + # following request sets Referer to test_start url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header @@ -366,15 +366,15 @@ with multiples lines Test whether errors happening anywhere in Crawler.crawl() are properly reported (and not somehow swallowed) after a graceful engine shutdown. The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.test_yield_seeds(), - SpiderMiddleware.process_test_yield_seeds(), etc. + spiders/middlewares/etc., e.g. raised in Spider.test_start(), + SpiderMiddleware.process_test_start(), etc. """ class TestError(Exception): pass class FaultySpider(SimpleSpider): - async def yield_seeds(self): + async def start(self): raise TestError crawler = get_crawler(FaultySpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index fa1c90880..42950ea94 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -153,7 +153,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -233,7 +233,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -313,7 +313,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -393,7 +393,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - async def yield_seeds(self): + async def start(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -580,7 +580,7 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - async def yield_seeds(self): + async def start(self): return yield diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index d485087ae..78c83ea83 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -28,7 +28,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } - async def yield_seeds(self): + async def start(self): self.times = {None: []} slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] diff --git a/tests/test_engine.py b/tests/test_engine.py index 223e89326..306a0e775 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -92,7 +92,7 @@ class MySpider(Spider): class DupeFilterSpider(MySpider): - async def yield_seeds(self): + async def start(self): for url in self.start_urls: yield Request(url) # no dont_filter=True diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index 93e4e701d..03a5ac64a 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -10,7 +10,7 @@ from scrapy.core.scheduler import BaseScheduler from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler -from .test_spider_yield_seeds import twisted_sleep +from .test_spider_start import twisted_sleep class MainTestCase(TestCase): @@ -93,7 +93,7 @@ class MainTestCase(TestCase): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) yield Request("data:,a") await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 19f260c0b..d658d1526 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -69,7 +69,7 @@ class AsyncDefNotAsyncioPipeline: class ItemSpider(Spider): name = "itemspider" - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index c2c1b6288..21b5675fd 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_test_yield_seeds(self, test_yield_seeds, spider): - for request in test_yield_seeds: + def process_test_start(self, test_start, spider): + for request in test_start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_test_yield_seeds"] = True + request.cb_kwargs["from_process_test_start"] = True yield request def process_spider_input(self, response, spider): @@ -62,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] - async def yield_seeds(self): + async def start(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -139,10 +139,10 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 2) def parse_spider_mw( - self, response, from_process_spider_input, from_process_test_yield_seeds + self, response, from_process_spider_input, from_process_test_start ): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_test_yield_seeds)) + self.checks.append(bool(from_process_test_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_signals.py b/tests/test_signals.py index 576843f53..de6cf459a 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -10,7 +10,7 @@ from tests.mockserver import MockServer class ItemSpider(Spider): name = "itemspider" - async def yield_seeds(self): + async def start(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} diff --git a/tests/test_spider_yield_seeds.py b/tests/test_spider_start.py similarity index 87% rename from tests/test_spider_yield_seeds.py rename to tests/test_spider_start.py index 8f2a11afb..34f1d5e66 100644 --- a/tests/test_spider_yield_seeds.py +++ b/tests/test_spider_start.py @@ -55,19 +55,19 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f - async def test_yield_seeds(self): + async def test_start(self): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_A await self._test_spider(TestSpider, [ITEM_A]) @deferred_f_from_coro_f - async def test_yield_seeds_subclass(self): + async def test_start_subclass(self): class BaseSpider(Spider): - async def yield_seeds(self): + async def start(self): yield ITEM_A class TestSpider(BaseSpider): @@ -104,7 +104,7 @@ class MainTestCase(TestCase): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_A def start_requests(self): @@ -115,7 +115,7 @@ class MainTestCase(TestCase): @deferred_f_from_coro_f async def test_universal_subclass(self): class BaseSpider(Spider): - async def yield_seeds(self): + async def start(self): yield ITEM_A def start_requests(self): @@ -126,28 +126,28 @@ class MainTestCase(TestCase): await self._test_spider(TestSpider, [ITEM_A]) - async def _test_yield_seeds(self, yield_seeds_, expected_items=None): + async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): name = "test" - yield_seeds = yield_seeds_ + start = start_ await self._test_spider(TestSpider, expected_items) @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_asyncio_delayed(self): - async def yield_seeds(spider): + async def start(spider): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) yield ITEM_A - await self._test_yield_seeds(yield_seeds, [ITEM_A]) + await self._test_start(start, [ITEM_A]) @deferred_f_from_coro_f async def test_twisted_delayed(self): - async def yield_seeds(spider): + async def start(spider): await maybe_deferred_to_future( twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) ) yield ITEM_A - await self._test_yield_seeds(yield_seeds, [ITEM_A]) + await self._test_start(start, [ITEM_A]) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9a699e8f6..74744548d 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_test_yield_seeds. + Should work for process_spider_output and, when it's supported, process_test_start. """ ITEM_TYPE: type | tuple @@ -321,12 +321,12 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): class ProcessYieldSeedsSimpleMiddleware: - def process_test_yield_seeds(self, test_yield_seeds, spider): - yield from test_yield_seeds + def process_test_start(self, test_start, spider): + yield from test_start class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): - """process_seeds tests for simple yield_seeds""" + """process_start tests for simple start""" ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware @@ -336,7 +336,7 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): for i in range(2): yield Request(f"https://example.com/{i}", dont_filter=True) yield {"name": "test item"} @@ -347,7 +347,7 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): ) self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - results = yield self.mwman.process_seeds(self.spider) + results = yield self.mwman.process_start(self.spider) return results @inlineCallbacks diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 5e3b2a014..fd2fc3581 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -30,7 +30,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - async def yield_seeds(self): + async def start(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index f157b8c4f..20efac543 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -36,7 +36,7 @@ class RecoverySpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -73,7 +73,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - async def yield_seeds(self): + async def start(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -83,7 +83,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - async def yield_seeds(self): + async def start(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -103,7 +103,7 @@ class GeneratorCallbackSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -140,7 +140,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -215,7 +215,7 @@ class GeneratorOutputChainSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -287,7 +287,7 @@ class NotGeneratorOutputChainSpider(Spider): }, } - async def yield_seeds(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_spidermiddleware_process_seeds.py b/tests/test_spidermiddleware_process_start.py similarity index 93% rename from tests/test_spidermiddleware_process_seeds.py rename to tests/test_spidermiddleware_process_start.py index 57f28f1fb..7d5cb7cb6 100644 --- a/tests/test_spidermiddleware_process_seeds.py +++ b/tests/test_spidermiddleware_process_start.py @@ -8,7 +8,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from .test_spider_yield_seeds import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep +from .test_spider_start import ASYNC_GEN_ERROR_MINIMUM_SECONDS, twisted_sleep ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} @@ -17,27 +17,27 @@ ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) async for seed in seeds: yield seed class NoOpSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): async for seed in seeds: yield seed class TwistedSleepSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)) async for seed in seeds: yield seed class UniversalSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): async for seed in seeds: yield seed @@ -51,14 +51,14 @@ class UniversalSpiderMiddleware: class ModernWrapSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_B class UniversalWrapSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_B def start_requests(self): @@ -73,7 +73,7 @@ class DeprecatedWrapSpider(Spider): class ModernWrapSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): yield ITEM_A async for seed in seeds: yield seed @@ -81,7 +81,7 @@ class ModernWrapSpiderMiddleware: class UniversalWrapSpiderMiddleware: - async def process_seeds(self, seeds): + async def process_start(self, seeds): yield ITEM_A async for seed in seeds: yield seed @@ -189,7 +189,7 @@ class MainTestCase(TestCase): class TestSpider(Spider): name = "test" - async def yield_seeds(self): + async def start(self): yield ITEM_A await self._test(spider_middlewares, TestSpider, [ITEM_A])