RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove unused code and privatize to protect against import of imports from public modules

This commit is contained in:
Adrian Chaves 2026-04-29 12:45:16 +02:00
parent 44f0fd0d37
commit 03f89c99ab
3 changed files with 25 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrapy/core/engine.py
View File

@ -29,7 +29,7 @@ from scrapy.exceptions import (
ScrapyDeprecationWarning,
)
from scrapy.http import Request, Response
from scrapy.utils._stopmode import StopMode, max_stop_mode, normalize_stop_mode
from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode, max_stop_mode
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
create_looping_call,
@ -120,7 +120,7 @@ class ExecutionEngine:
self.running: bool = False
self._starting: bool = False
self._stopping: bool = False
self._stop_mode: StopMode = "graceful"
self._stop_mode: _StopMode = "graceful"
self._downloader_fast_stopped: bool = False
self.paused: bool = False
self._spider_closed_callback: Callable[
@ -206,7 +206,7 @@ class ExecutionEngine:
await maybe_deferred_to_future(self._closewait)
def stop(
self, *, mode: StopMode = "graceful"
self, *, mode: _StopMode = "graceful"
) -> Deferred[None]: # pragma: no cover
warnings.warn(
"ExecutionEngine.stop() is deprecated, use stop_async() instead",
@ -215,13 +215,13 @@ class ExecutionEngine:
)
return deferred_from_coro(self.stop_async(mode=mode))
async def stop_async(self, *, mode: StopMode = "graceful") -> None:
async def stop_async(self, *, mode: _StopMode = "graceful") -> None:
"""Gracefully stop the execution engine.
.. versionadded:: 2.14
"""
mode = normalize_stop_mode(mode, allow_force=False)
mode = _normalize_stop_mode(mode, allow_force=False)
if not self._starting and not self._stopping:
raise RuntimeError("Engine not running")
@ -613,7 +613,7 @@ class ExecutionEngine:
self,
spider: Spider,
reason: str = "cancelled",
mode: StopMode = "graceful",
mode: _StopMode = "graceful",
) -> Deferred[None]: # pragma: no cover
warnings.warn(
"ExecutionEngine.close_spider() is deprecated, use close_spider_async() instead",
@ -646,13 +646,13 @@ class ExecutionEngine:
self,
*,
reason: str = "cancelled",
mode: StopMode = "graceful",
mode: _StopMode = "graceful",
) -> None:
"""Close (cancel) spider and clear all its outstanding requests.
.. versionadded:: 2.14
"""
mode = normalize_stop_mode(mode, allow_force=False)
mode = _normalize_stop_mode(mode, allow_force=False)
self._stop_mode = max_stop_mode(self._stop_mode, mode)
if self.spider is None:

26
scrapy/crawler.py
View File

@ -19,7 +19,7 @@ from scrapy.extension import ExtensionManager
from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
from scrapy.utils._stopmode import StopMode, normalize_stop_mode
from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
from scrapy.utils.log import (
configure_logging,
@ -242,7 +242,7 @@ class Crawler:
def _create_engine(self) -> ExecutionEngine:
return ExecutionEngine(self, lambda _: self.stop_async())
def stop(self, *, mode: StopMode = "graceful") -> Deferred[None]:
def stop(self, *, mode: _StopMode = "graceful") -> Deferred[None]:
"""Start a graceful stop of the crawler and return a deferred that is
fired when the crawler is stopped."""
warnings.warn(
@ -252,12 +252,12 @@ class Crawler:
)
return deferred_from_coro(self.stop_async(mode=mode))
async def stop_async(self, *, mode: StopMode = "graceful") -> None:
async def stop_async(self, *, mode: _StopMode = "graceful") -> None:
"""Start a graceful stop of the crawler and complete when the crawler is stopped.
.. versionadded:: 2.14
"""
mode = normalize_stop_mode(mode)
mode = _normalize_stop_mode(mode)
was_crawling = self.crawling
self.crawling = False
@ -506,13 +506,13 @@ class CrawlerRunner(CrawlerRunnerBase):
self._active.discard(d)
self.bootstrap_failed |= not getattr(crawler, "spider", None) or failed
def stop(self, *, mode: StopMode = "graceful") -> Deferred[Any]:
def stop(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:
"""
Stops simultaneously all the crawling jobs taking place.
Returns a deferred that is fired when they all have ended.
"""
mode = normalize_stop_mode(mode)
mode = _normalize_stop_mode(mode)
return DeferredList(
deferred_from_coro(c.stop_async(mode=mode)) for c in self.crawlers
)
@ -631,13 +631,13 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
task.add_done_callback(_done)
return task
async def stop(self, *, mode: StopMode = "graceful") -> None:
async def stop(self, *, mode: _StopMode = "graceful") -> None:
"""
Stops simultaneously all the crawling jobs taking place.
Completes when they all have ended.
"""
mode = normalize_stop_mode(mode)
mode = _normalize_stop_mode(mode)
if self.crawlers:
await asyncio.wait(
[asyncio.create_task(c.stop_async(mode=mode)) for c in self.crawlers]
@ -760,7 +760,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):
)
@abstractmethod
def _stop_dfd(self, *, mode: StopMode = "graceful") -> Deferred[Any]:
def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:
raise NotImplementedError
@inlineCallbacks
@ -830,7 +830,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
crawler._set_force_stop_callback(self._force_stop)
return crawler
def _stop_dfd(self, *, mode: StopMode = "graceful") -> Deferred[Any]:
def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:
return self.stop(mode=mode)
def start(
@ -938,7 +938,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
if (task := self._reactorless_main_task) is not None:
loop.call_soon_threadsafe(task.cancel)
def _stop_dfd(self, *, mode: StopMode = "graceful") -> Deferred[Any]:
def _stop_dfd(self, *, mode: _StopMode = "graceful") -> Deferred[Any]:
return deferred_from_coro(self.stop(mode=mode))
def start(
@ -1084,7 +1084,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
}
)
def _schedule_reactorless_shutdown(self, *, mode: StopMode) -> None:
def _schedule_reactorless_shutdown(self, *, mode: _StopMode) -> None:
if (loop := self._reactorless_loop) is None:
return
@ -1107,7 +1107,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
self._log_fast_shutdown(signum)
self._schedule_reactorless_shutdown(mode="fast")
async def _shutdown_reactorless(self, *, mode: StopMode) -> None:
async def _shutdown_reactorless(self, *, mode: _StopMode) -> None:
await self.stop(mode=mode)
if not self._stop_after_crawl:
# wait until crawl tasks finish and cancel the future

10
scrapy/utils/_stopmode.py
View File

@ -2,18 +2,16 @@ from __future__ import annotations
from typing import Literal
StopMode = Literal["graceful", "fast", "force"]
_StopMode = Literal["graceful", "fast", "force"]
_STOP_MODE_PRIORITY: dict[StopMode, int] = {
_STOP_MODE_PRIORITY: dict[_StopMode, int] = {
"graceful": 0,
"fast": 1,
"force": 2,
}
def normalize_stop_mode(mode: StopMode | None, *, allow_force: bool = True) -> StopMode:
if mode is None:
return "graceful"
def _normalize_stop_mode(mode: _StopMode, *, allow_force: bool = True) -> _StopMode:
if mode not in _STOP_MODE_PRIORITY:
raise ValueError(
f"Unknown stop mode {mode!r}. Expected one of: graceful, fast, force"
@ -23,7 +21,7 @@ def normalize_stop_mode(mode: StopMode | None, *, allow_force: bool = True) -> S
return mode
def max_stop_mode(mode1: StopMode, mode2: StopMode) -> StopMode:
def max_stop_mode(mode1: _StopMode, mode2: _StopMode) -> _StopMode:
if _STOP_MODE_PRIORITY[mode1] >= _STOP_MODE_PRIORITY[mode2]:
return mode1
return mode2