diff --git a/docs/news.rst b/docs/news.rst index 28ee6f023..972e91b97 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -15,9 +15,22 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``scrapy.core.engine.Slot.start_requests` and its matching - ``Slot.__init__()`` parameter have been removed, replaced by - ``seeds_iterator``. +- The second parameter of + ``scrapy.core.engine.ExecutionEngine.open_spider()``, ``start_requests``, + has been removed. The starting requests are determined by the ``spider`` + parameter instead. + +- ``scrapy.core.spidermw.SpiderMiddlewareManager.process_start_requests()`` + has been replaced by + ``scrapy.core.spidermw.SpiderMiddlewareManager.process_seeds()``. + +- ``scrapy.core.engine.Slot`` has been renamed to + ``scrapy.core.engine._Slot`` and should not be used. + +- ``scrapy.core.engine.ExecutionEngine.slot`` has been renamed to + ``scrapy.core.engine.ExecutionEngine._slot`` and should not be used. + +- The ``slot`` :ref:`telnet variable ` has been removed. Deprecations ~~~~~~~~~~~~ diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 0e4a8fa6c..3e9bbe56e 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -59,6 +59,8 @@ Default Username and Password can be overridden by the settings You need the telnet program which comes installed by default in Windows, and most Linux distros. +.. _telnet-vars: + Available variables in the telnet console ========================================= @@ -77,8 +79,6 @@ convenience: +----------------+-------------------------------------------------------------------+ | ``spider`` | the active spider | +----------------+-------------------------------------------------------------------+ -| ``slot`` | the engine slot | -+----------------+-------------------------------------------------------------------+ | ``extensions`` | the Extension Manager (Crawler.extensions attribute) | +----------------+-------------------------------------------------------------------+ | ``stats`` | the Stats Collector (Crawler.stats attribute) | @@ -114,10 +114,10 @@ using the telnet console:: engine.scraper.is_idle() : False engine.spider.name : followall engine.spider_is_idle() : False - engine.slot.closing : False - len(engine.slot.inprogress) : 16 - len(engine.slot.scheduler.dqs or []) : 0 - len(engine.slot.scheduler.mqs) : 92 + engine._slot.closing : False + len(engine._slot.inprogress) : 16 + len(engine._slot.scheduler.dqs or []) : 0 + len(engine._slot.scheduler.mqs) : 92 len(engine.scraper.slot.queue) : 0 len(engine.scraper.slot.active) : 0 engine.scraper.slot.active_size : 0 diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 69a09d2b6..df1bf8767 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -35,6 +35,10 @@ class QPSSpider(Spider): self.download_delay = float(self.download_delay) async def yield_seeds(self): + for seed in self.start_requests(): + yield seed + + def start_requests(self): url = self.benchurl if self.latency is not None: url += f"?latency={self.latency}" diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 18eeebb7d..24dfc0106 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -81,8 +81,7 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader async def yield_seeds(self): - requests = conman.from_spider(self, self._result) - for request in requests: + for request in conman.from_spider(self, self._result): yield request with set_environ(SCRAPY_CHECK="true"): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 60d959b2a..7cdc6287f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -42,14 +42,12 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -class Slot: +class _Slot: def __init__( self, close_if_idle: bool, nextcall: CallLaterOnce[Deferred[None]], scheduler: BaseScheduler, - *, - seeds: AsyncIterator[Any] | None, ) -> None: self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() @@ -57,7 +55,6 @@ class Slot: self.nextcall: CallLaterOnce[Deferred[None]] = nextcall self.scheduler: BaseScheduler = scheduler self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) - self.seeds: AsyncIterator[Any] | None = seeds def add_request(self, request: Request) -> None: self.inprogress.add(request) @@ -100,7 +97,7 @@ class ExecutionEngine: self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Slot | None = None + self._slot: _Slot | None = None self.spider: Spider | None = None self.running: bool = False self.paused: bool = False @@ -115,6 +112,7 @@ class ExecutionEngine: ) self.start_time: float | None = None self._load_seeding_policy() + self._seeds: AsyncIterator[Any] | None = None def _load_seeding_policy(self) -> None: try: @@ -195,16 +193,16 @@ class ExecutionEngine: @inlineCallbacks def _process_next_seed(self): try: - seed = yield deferred_from_coro(self.slot.seeds.__anext__()) + seed = yield deferred_from_coro(self._seeds.__anext__()) except StopAsyncIteration: - self.slot.seeds = None + self._seeds = None except RuntimeError: # “RuntimeError: anext(): asynchronous generator is already # running” happens if yield_seeds is taking long to yield the # next seed. pass except Exception: - self.slot.seeds = None + self._seeds = None logger.error( "Error while reading seeds", exc_info=True, @@ -215,33 +213,33 @@ class ExecutionEngine: self.crawl(seed) else: self.scraper.start_itemproc(seed, response=None) - self.slot.nextcall.schedule() + self._slot.nextcall.schedule() @inlineCallbacks def _start_next_requests(self) -> Generator[Deferred[Any], Any, None]: - if self.slot is None or self.paused: + if self._slot is None or self._slot.closing is not None or self.paused: return self._start_scheduled_requests() - if self.slot.seeds is not None and not self._needs_backout(): + if self._seeds is not None and not self._needs_backout(): yield self._process_next_seed() - if self.spider_is_idle() and self.slot.close_if_idle: + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() def _needs_backout(self) -> bool: - assert self.slot is not None # typing + assert self._slot is not None # typing assert self.scraper.slot is not None # typing return ( not self.running - or bool(self.slot.closing) + or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() ) def _start_scheduled_request(self) -> Deferred[None] | None: - assert self.slot is not None # typing + assert self._slot is not None # typing assert self.spider is not None # typing - request = self.slot.scheduler.next_request() + request = self._slot.scheduler.next_request() if request is None: return None @@ -256,8 +254,8 @@ class ExecutionEngine: ) def _remove_request(_: Any) -> None: - assert self.slot - self.slot.remove_request(request) + assert self._slot + self._slot.remove_request(request) d2: Deferred[None] = d.addBoth(_remove_request) d2.addErrback( @@ -267,7 +265,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - slot = self.slot + slot = self._slot d2.addBoth(lambda _: slot.nextcall.schedule()) d2.addErrback( lambda f: logger.info( @@ -304,22 +302,22 @@ class ExecutionEngine: return d def spider_is_idle(self) -> bool: - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False if self.downloader.active: # downloader has pending requests return False - if self.slot.seeds is not None: # not all start requests are handled + if self._seeds is not None: # not all start requests are handled return False - return not self.slot.scheduler.has_pending_requests() + return not self._slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request, self.spider) - self.slot.nextcall.schedule() # type: ignore[union-attr] + self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request, spider: Spider) -> None: request_scheduled_result = self.signals.send_catch_log( @@ -331,7 +329,7 @@ class ExecutionEngine: for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return - if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=spider ) @@ -348,14 +346,14 @@ class ExecutionEngine: def _downloaded( self, result: Response | Request | Failure, request: Request ) -> Deferred[Response] | Response | Failure: - assert self.slot is not None # typing - self.slot.remove_request(request) + assert self._slot is not None # typing + self._slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result def _download(self, request: Request) -> Deferred[Response | Request]: - assert self.slot is not None # typing + assert self._slot is not None # typing - self.slot.add_request(request) + self._slot.add_request(request) def _on_success(result: Response | Request) -> Response | Request: if not isinstance(result, (Response, Request)): @@ -380,8 +378,8 @@ class ExecutionEngine: return result def _on_complete(_: _T) -> _T: - assert self.slot is not None - self.slot.nextcall.schedule() + assert self._slot is not None + self._slot.nextcall.schedule() return _ assert self.spider is not None @@ -396,13 +394,13 @@ class ExecutionEngine: spider: Spider, close_if_idle: bool = True, ) -> Generator[Deferred[Any], Any, None]: - if self.slot is not None: + if self._slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._start_next_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - seeds = yield self.scraper.spidermw.process_seeds(spider) - self.slot = Slot(close_if_idle, nextcall, scheduler, seeds=seeds) + self._seeds = yield self.scraper.spidermw.process_seeds(spider) + self._slot = _Slot(close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open") and (d := scheduler.open(spider)): yield d @@ -410,8 +408,8 @@ class ExecutionEngine: assert self.crawler.stats self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - self.slot.nextcall.schedule() - self.slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) def _spider_idle(self) -> None: """ @@ -440,17 +438,17 @@ class ExecutionEngine: def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") - if self.slot.closing is not None: - return self.slot.closing + if self._slot.closing is not None: + return self._slot.closing logger.info( "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - dfd = self.slot.close() + dfd = self._slot.close() def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: @@ -466,8 +464,8 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure("Scraper close failure")) - if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) + if hasattr(self._slot.scheduler, "close"): + dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7c9932770..280c38ccd 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager): super().__init__(*middlewares) def _check_deprecated_process_start_requests_use( - self, middlewares: list[Any] + self, middlewares: tuple[Any] ) -> None: deprecated_middlewares = [ middleware @@ -374,9 +374,11 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, AsyncIterator[Any]]: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: - seeds = iter(spider.start_requests()) - seeds = yield self._process_chain("process_start_requests", seeds, spider) - seeds = as_async_generator(seeds) + sync_seeds = iter(spider.start_requests()) + sync_seeds = yield self._process_chain( + "process_start_requests", sync_seeds, spider + ) + seeds = as_async_generator(sync_seeds) else: seeds = yield self._iter_seeds(spider) seeds = yield self._process_chain("process_seeds", seeds) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ac832e025..bacee8f0a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -104,7 +104,6 @@ class TelnetConsole(protocol.ServerFactory): telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, - "slot": self.crawler.engine.slot, "crawler": self.crawler, "extensions": self.crawler.extensions, "stats": self.crawler.stats, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index b8d360d81..85bda1bbb 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -40,7 +40,7 @@ class Spider(object_ref): custom_settings: dict[_SettingsKeyT, Any] | None = None #: Seed URLs. See :meth:`yield_seeds`. - start_urls: list[str] = [] + start_urls: list[str] def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: @@ -48,6 +48,8 @@ class Spider(object_ref): elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) + if not hasattr(self, "start_urls"): + self.start_urls: list[str] = [] @property def logger(self) -> SpiderLoggerAdapter: diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index d88b109b2..5c84ae5fe 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -30,7 +30,7 @@ class InitSpider(Spider): ) async def yield_seeds(self) -> AsyncIterator[Any]: - async for seed in super().yield_seeds(): + for seed in self.start_requests(): yield seed def start_requests(self) -> Iterable[Request]: diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 291f5ee25..1be23421c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -54,7 +54,7 @@ class SitemapSpider(Spider): self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] async def yield_seeds(self) -> AsyncIterator[Any]: - async for seed in super().yield_seeds(): + for seed in self.start_requests(): yield seed def start_requests(self) -> Iterable[Request]: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index b6ab156a3..8b8ab927d 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -46,8 +46,6 @@ class ${ProjectName}SpiderMiddleware: async def process_seeds(self, seeds): # Called with the seeds from the spider yield_seeds() method or with # the output of the maching method of an earlier spider middleware. - - # Must return only requests (not items). async for seed in seeds: yield seed diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 52f29e22c..1e0c53212 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -18,10 +18,10 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: "engine.scraper.is_idle()", "engine.spider.name", "engine.spider_is_idle()", - "engine.slot.closing", - "len(engine.slot.inprogress)", - "len(engine.slot.scheduler.dqs or [])", - "len(engine.slot.scheduler.mqs)", + "engine._slot.closing", + "len(engine._slot.inprogress)", + "len(engine._slot.scheduler.dqs or [])", + "len(engine._slot.scheduler.mqs)", "len(engine.scraper.slot.queue)", "len(engine.scraper.slot.active)", "engine.scraper.slot.active_size", diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index 686c35162..6076211ec 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -15,7 +15,8 @@ class NoRequestsSpider(scrapy.Spider): async def yield_seeds(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index e814a28d9..13c6fff85 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index a8222b01f..ff2e47a70 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f7e96ce16..9959f31fe 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -9,7 +9,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 09d81593d..59feced94 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -16,7 +16,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index 8d71efccd..f297fd0d7 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -17,7 +17,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index a3ca7b5bf..65f5e033f 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 9b3fa65eb..a221764d2 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -9,7 +9,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 67125247f..a0aff999b 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -9,7 +9,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index d43416d53..6ac1043b5 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -11,7 +11,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index d1079b1a3..f7352af57 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -18,7 +18,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index 4f00514c7..1071e453d 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -10,7 +10,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index a61cadd06..3773092b0 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index 67c56cbc8..bdc217fde 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -11,7 +11,8 @@ class NoRequestsSpider(Spider): } async def yield_seeds(self): - return [] + return + yield configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) diff --git a/tests/test_commands.py b/tests/test_commands.py index 874ecf93c..84ac8b125 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -672,7 +672,8 @@ class MySpider(scrapy.Spider): async def yield_seeds(self): self.logger.debug("It Works!") - return [] + return + yield """ badspider = """ @@ -848,7 +849,8 @@ class MySpider(scrapy.Spider): async def yield_seeds(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -868,7 +870,8 @@ class MySpider(scrapy.Spider): json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -889,7 +892,8 @@ class MySpider(scrapy.Spider): name = 'myspider' async def yield_seeds(self): - return [] + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) @@ -906,7 +910,8 @@ class MySpider(scrapy.Spider): async def yield_seeds(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) @@ -985,7 +990,8 @@ class MySpider(scrapy.Spider): async def yield_seeds(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield """ args = ["-a", "foo=42"] log = self.get_log(spider_code, args=args) @@ -1055,7 +1061,8 @@ class MySpider(scrapy.Spider): async def yield_seeds(self): self.logger.debug('It works!') - return [] + return + yield """ log = self.get_log(spider_code) assert "[myspider] DEBUG: It works!" in log @@ -1069,7 +1076,8 @@ class MySpider(scrapy.Spider): async def yield_seeds(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -1089,7 +1097,8 @@ class MySpider(scrapy.Spider): json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -1110,7 +1119,8 @@ class MySpider(scrapy.Spider): name = 'myspider' async def yield_seeds(self): - return [] + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 51f80a67e..ad2db4128 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -581,7 +581,8 @@ class NoRequestsSpider(scrapy.Spider): name = "no_request" async def yield_seeds(self): - return [] + return + yield @pytest.mark.usefixtures("reactor_pytest") diff --git a/tests/test_engine.py b/tests/test_engine.py index 3fa7d889f..223e89326 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -29,7 +29,7 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request @@ -498,7 +498,8 @@ def test_request_scheduled_signal(caplog): return yield - engine.slot = Slot(None, Mock(), scheduler, seeds=seeds) + engine._seeds = seeds() + engine._slot = _Slot(False, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") engine._schedule_request(keep_request, spider) diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index 29c68c8e3..c74dc3ca1 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -98,10 +98,10 @@ class MainTestCase(TestCase): await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) yield Request("data:,a") await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) - self.crawler.engine.slot.scheduler.enqueue_request(Request("data:,b")) + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) await maybe_deferred_to_future(twisted_sleep(sleep_seconds)) yield Request("data:,c") - self.crawler.engine.slot.scheduler.stop = True + self.crawler.engine._slot.scheduler.stop = True def parse(self, response): pass