diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 973ae3258..d9e0c5eed 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -29,7 +29,7 @@ from scrapy.exceptions import ( ScrapyDeprecationWarning, ) from scrapy.http import Request, Response -from scrapy.utils._stopmode import StopMode, max_stop_mode, normalize_stop_mode +from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode, max_stop_mode from scrapy.utils.asyncio import ( AsyncioLoopingCall, create_looping_call, @@ -120,7 +120,7 @@ class ExecutionEngine: self.running: bool = False self._starting: bool = False self._stopping: bool = False - self._stop_mode: StopMode = "graceful" + self._stop_mode: _StopMode = "graceful" self._downloader_fast_stopped: bool = False self.paused: bool = False self._spider_closed_callback: Callable[ @@ -206,7 +206,7 @@ class ExecutionEngine: await maybe_deferred_to_future(self._closewait) def stop( - self, *, mode: StopMode = "graceful" + self, *, mode: _StopMode = "graceful" ) -> Deferred[None]: # pragma: no cover warnings.warn( "ExecutionEngine.stop() is deprecated, use stop_async() instead", @@ -215,13 +215,13 @@ class ExecutionEngine: ) return deferred_from_coro(self.stop_async(mode=mode)) - async def stop_async(self, *, mode: StopMode = "graceful") -> None: + async def stop_async(self, *, mode: _StopMode = "graceful") -> None: """Gracefully stop the execution engine. .. versionadded:: 2.14 """ - mode = normalize_stop_mode(mode, allow_force=False) + mode = _normalize_stop_mode(mode, allow_force=False) if not self._starting and not self._stopping: raise RuntimeError("Engine not running") @@ -613,7 +613,7 @@ class ExecutionEngine: self, spider: Spider, reason: str = "cancelled", - mode: StopMode = "graceful", + mode: _StopMode = "graceful", ) -> Deferred[None]: # pragma: no cover warnings.warn( "ExecutionEngine.close_spider() is deprecated, use close_spider_async() instead", @@ -646,13 +646,13 @@ class ExecutionEngine: self, *, reason: str = "cancelled", - mode: StopMode = "graceful", + mode: _StopMode = "graceful", ) -> None: """Close (cancel) spider and clear all its outstanding requests. .. versionadded:: 2.14 """ - mode = normalize_stop_mode(mode, allow_force=False) + mode = _normalize_stop_mode(mode, allow_force=False) self._stop_mode = max_stop_mode(self._stop_mode, mode) if self.spider is None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ace5f4237..7fd39571e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -19,7 +19,7 @@ from scrapy.extension import ExtensionManager from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader -from scrapy.utils._stopmode import StopMode, normalize_stop_mode +from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.log import ( configure_logging, @@ -242,7 +242,7 @@ class Crawler: def _create_engine(self) -> ExecutionEngine: return ExecutionEngine(self, lambda _: self.stop_async()) - def stop(self, *, mode: StopMode = "graceful") -> Deferred[None]: + def stop(self, *, mode: _StopMode = "graceful") -> Deferred[None]: """Start a graceful stop of the crawler and return a deferred that is fired when the crawler is stopped.""" warnings.warn( @@ -252,12 +252,12 @@ class Crawler: ) return deferred_from_coro(self.stop_async(mode=mode)) - async def stop_async(self, *, mode: StopMode = "graceful") -> None: + async def stop_async(self, *, mode: _StopMode = "graceful") -> None: """Start a graceful stop of the crawler and complete when the crawler is stopped. .. versionadded:: 2.14 """ - mode = normalize_stop_mode(mode) + mode = _normalize_stop_mode(mode) was_crawling = self.crawling self.crawling = False @@ -506,13 +506,13 @@ class CrawlerRunner(CrawlerRunnerBase): self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) or failed - def stop(self, *, mode: StopMode = "graceful") -> Deferred[Any]: + def stop(self, *, mode: _StopMode = "graceful") -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - mode = normalize_stop_mode(mode) + mode = _normalize_stop_mode(mode) return DeferredList( deferred_from_coro(c.stop_async(mode=mode)) for c in self.crawlers ) @@ -631,13 +631,13 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): task.add_done_callback(_done) return task - async def stop(self, *, mode: StopMode = "graceful") -> None: + async def stop(self, *, mode: _StopMode = "graceful") -> None: """ Stops simultaneously all the crawling jobs taking place. Completes when they all have ended. """ - mode = normalize_stop_mode(mode) + mode = _normalize_stop_mode(mode) if self.crawlers: await asyncio.wait( [asyncio.create_task(c.stop_async(mode=mode)) for c in self.crawlers] @@ -760,7 +760,7 @@ class CrawlerProcessBase(CrawlerRunnerBase): ) @abstractmethod - def _stop_dfd(self, *, mode: StopMode = "graceful") -> Deferred[Any]: + def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]: raise NotImplementedError @inlineCallbacks @@ -830,7 +830,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): crawler._set_force_stop_callback(self._force_stop) return crawler - def _stop_dfd(self, *, mode: StopMode = "graceful") -> Deferred[Any]: + def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]: return self.stop(mode=mode) def start( @@ -938,7 +938,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): if (task := self._reactorless_main_task) is not None: loop.call_soon_threadsafe(task.cancel) - def _stop_dfd(self, *, mode: StopMode = "graceful") -> Deferred[Any]: + def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]: return deferred_from_coro(self.stop(mode=mode)) def start( @@ -1084,7 +1084,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): } ) - def _schedule_reactorless_shutdown(self, *, mode: StopMode) -> None: + def _schedule_reactorless_shutdown(self, *, mode: _StopMode) -> None: if (loop := self._reactorless_loop) is None: return @@ -1107,7 +1107,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): self._log_fast_shutdown(signum) self._schedule_reactorless_shutdown(mode="fast") - async def _shutdown_reactorless(self, *, mode: StopMode) -> None: + async def _shutdown_reactorless(self, *, mode: _StopMode) -> None: await self.stop(mode=mode) if not self._stop_after_crawl: # wait until crawl tasks finish and cancel the future diff --git a/scrapy/utils/_stopmode.py b/scrapy/utils/_stopmode.py index 05b64ec9a..6acd1c42e 100644 --- a/scrapy/utils/_stopmode.py +++ b/scrapy/utils/_stopmode.py @@ -2,18 +2,16 @@ from __future__ import annotations from typing import Literal -StopMode = Literal["graceful", "fast", "force"] +_StopMode = Literal["graceful", "fast", "force"] -_STOP_MODE_PRIORITY: dict[StopMode, int] = { +_STOP_MODE_PRIORITY: dict[_StopMode, int] = { "graceful": 0, "fast": 1, "force": 2, } -def normalize_stop_mode(mode: StopMode | None, *, allow_force: bool = True) -> StopMode: - if mode is None: - return "graceful" +def _normalize_stop_mode(mode: _StopMode, *, allow_force: bool = True) -> _StopMode: if mode not in _STOP_MODE_PRIORITY: raise ValueError( f"Unknown stop mode {mode!r}. Expected one of: graceful, fast, force" @@ -23,7 +21,7 @@ def normalize_stop_mode(mode: StopMode | None, *, allow_force: bool = True) -> S return mode -def max_stop_mode(mode1: StopMode, mode2: StopMode) -> StopMode: +def max_stop_mode(mode1: _StopMode, mode2: _StopMode) -> _StopMode: if _STOP_MODE_PRIORITY[mode1] >= _STOP_MODE_PRIORITY[mode2]: return mode1 return mode2