diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 7ff8d3464..598edfeb5 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -35,6 +35,13 @@ how you :ref:`configure the downloader middlewares :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. + The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`, + :attr:`request_fingerprinter` and :attr:`stats` attributes get their value + when the crawl starts, and raise :exc:`RuntimeError` when read before that. + + .. versionchanged:: VERSION + Those attributes used to be ``None`` before getting their value. + .. attribute:: request_fingerprinter The request fingerprint builder of this crawler. diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 51caed57f..b6255ec7e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -281,7 +281,6 @@ class Command(BaseRunSpiderCommand): ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: - assert self.pcrawler.engine itemproc = self.pcrawler.engine.scraper.itemproc if hasattr(itemproc, "process_item_async"): for item in items: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7a1e0fcd..b63d136a4 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -112,7 +112,6 @@ class ExecutionEngine: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals - assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter self._slot: _Slot | None = None self.spider: Spider | None = None @@ -299,7 +298,6 @@ class ExecutionEngine: response=None, spider=self.spider, ) - assert self.crawler.stats self.crawler.stats.inc_value("spider_exceptions/count") self.crawler.stats.inc_value( f"spider_exceptions/{type(exception).__name__}" @@ -563,17 +561,17 @@ class ExecutionEngine: if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): await maybe_deferred_to_future(d) await self.scraper.open_spider_async() - assert self.crawler.stats - if argument_is_required(self.crawler.stats.open_spider, "spider"): + stats = self.crawler.stats + if argument_is_required(stats.open_spider, "spider"): warnings.warn( - f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument," f" this is deprecated and the argument will not be passed in future Scrapy versions.", ScrapyDeprecationWarning, stacklevel=2, ) - self.crawler.stats.open_spider(spider=self.crawler.spider) + stats.open_spider(spider=self.crawler.spider) else: - self.crawler.stats.open_spider() + stats.open_spider() await self.signals.send_catch_log_async( signals.spider_opened, spider=self.crawler.spider ) @@ -676,20 +674,18 @@ class ExecutionEngine: extra={"spider": spider}, ) - assert self.crawler.stats try: - if argument_is_required(self.crawler.stats.close_spider, "spider"): + stats = self.crawler.stats + if argument_is_required(stats.close_spider, "spider"): warnings.warn( - f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f"The close_spider() method of {global_object_name(type(stats))} requires a spider argument," f" this is deprecated and the argument will not be passed in future Scrapy versions.", ScrapyDeprecationWarning, stacklevel=2, ) - self.crawler.stats.close_spider( - spider=self.crawler.spider, reason=reason - ) + stats.close_spider(spider=self.crawler.spider, reason=reason) else: - self.crawler.stats.close_spider(reason=reason) + stats.close_spider(reason=reason) except Exception: logger.error("Stats close failure") diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 58e37ce5e..6426b1751 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -120,7 +120,6 @@ class Scraper: self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler: Crawler = crawler self.signals: SignalManager = crawler.signals - assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter def _check_deprecated_itemproc_method(self, method: str) -> None: @@ -355,7 +354,6 @@ class Scraper: assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): - assert self.crawler.engine is not None # typing _schedule_coro( self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled") ) @@ -374,11 +372,9 @@ class Scraper: response=response, spider=self.crawler.spider, ) - assert self.crawler.stats - self.crawler.stats.inc_value("spider_exceptions/count") - self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}" - ) + stats = self.crawler.stats + stats.inc_value("spider_exceptions/count") + stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}") def handle_spider_output( self, @@ -456,7 +452,6 @@ class Scraper: Items are sent to the item pipelines, requests are scheduled. """ if isinstance(output, Request): - assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) return if output is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e2f726519..44c4ffdcf 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -8,7 +8,7 @@ import signal import warnings from abc import ABC, abstractmethod from functools import partial -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks @@ -58,7 +58,55 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") +class _LateAttribute(Generic[_T]): + """Descriptor for a :class:`Crawler` attribute that only gets a value once + the crawl starts. + + The value is kept in an attribute of the same name prefixed with an + underscore, and reading it before it is set raises :exc:`RuntimeError`. + This way the public attribute can be annotated as always set, and its + users, both in Scrapy and in third-party code, do not need to narrow its + type on every use. Code that runs before the crawl starts reads the + underscore-prefixed attribute instead. + """ + + def __set_name__(self, owner: type[Crawler], name: str) -> None: + self._name = name + self._private_name = f"_{name}" + + @overload + def __get__(self, instance: None, owner: type[Crawler]) -> _LateAttribute[_T]: ... + + @overload + def __get__(self, instance: Crawler, owner: type[Crawler]) -> _T: ... + + def __get__( + self, instance: Crawler | None, owner: type[Crawler] + ) -> _LateAttribute[_T] | _T: + if instance is None: + return self + value: _T | None = getattr(instance, self._private_name) + if value is None: + raise RuntimeError( + f"Crawler.{self._name} is not set yet. It is set when the " + "crawl starts, so it can only be used from then on, e.g. " + "from the spider_opened signal handler onwards." + ) + return value + + def __set__(self, instance: Crawler, value: _T) -> None: + setattr(instance, self._private_name, value) + + class Crawler: + engine: _LateAttribute[ExecutionEngine] = _LateAttribute() + extensions: _LateAttribute[ExtensionManager] = _LateAttribute() + logformatter: _LateAttribute[LogFormatter] = _LateAttribute() + request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = ( + _LateAttribute() + ) + stats: _LateAttribute[StatsCollector] = _LateAttribute() + def __init__( self, spidercls: type[Spider], @@ -83,12 +131,13 @@ class Crawler: self.crawling: bool = False self._started: bool = False - self.extensions: ExtensionManager | None = None - self.stats: StatsCollector | None = None - self.logformatter: LogFormatter | None = None - self.request_fingerprinter: RequestFingerprinterProtocol | None = None self.spider: Spider | None = None - self.engine: ExecutionEngine | None = None + + self._engine: ExecutionEngine | None = None + self._extensions: ExtensionManager | None = None + self._logformatter: LogFormatter | None = None + self._request_fingerprinter: RequestFingerprinterProtocol | None = None + self._stats: StatsCollector | None = None def _update_root_log_handler(self) -> None: if get_scrapy_root_handler() is not None: @@ -225,8 +274,8 @@ class Crawler: yield deferred_from_coro(self.engine.start_async()) except Exception: self.crawling = False - if self.engine is not None: - yield deferred_from_coro(self.engine.close_async()) + if self._engine is not None: + yield deferred_from_coro(self._engine.close_async()) raise async def crawl_async(self, *args: Any, **kwargs: Any) -> None: @@ -255,8 +304,8 @@ class Crawler: await self.engine.start_async() except Exception: self.crawling = False - if self.engine is not None: - await self.engine.close_async() + if self._engine is not None: + await self._engine.close_async() raise def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: @@ -282,7 +331,6 @@ class Crawler: """ if self.crawling: self.crawling = False - assert self.engine if self.engine.running: await self.engine.stop_async() @@ -313,7 +361,7 @@ class Crawler: This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.engine: + if self._engine is None: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " "the crawl engine has been created." @@ -331,7 +379,7 @@ class Crawler: created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.extensions: + if self._extensions is None: raise RuntimeError( "Crawler.get_extension() can only be called after the " "extension manager has been created." @@ -348,7 +396,7 @@ class Crawler: This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.engine: + if self._engine is None: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " "crawl engine has been created." @@ -365,7 +413,7 @@ class Crawler: This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.engine: + if self._engine is None: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " "crawl engine has been created." diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index e7ca0ac0e..3ebd98027 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -55,7 +55,6 @@ class HttpCacheMiddleware: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.stats o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 0d5c9e4c9..b9a26df3a 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -61,7 +61,6 @@ class OffsiteMiddleware: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index f910d07c8..dd2897ee4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -94,7 +94,6 @@ def get_retry_request( retry-related job stats """ settings = spider.crawler.settings - assert spider.crawler.stats stats = spider.crawler.stats retry_times = request.meta.get("retry_times", 0) + 1 if max_retry_times is None: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 81a3a887f..d7aa8738c 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler from scrapy.robotstxt import RobotParser + from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ class RobotsTxtMiddleware: self._default_useragent: str = crawler.settings["USER_AGENT"] self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler + self._stats: StatsCollector = crawler.stats self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") @@ -78,8 +80,7 @@ class RobotsTxtMiddleware: {"request": request}, extra={"spider": self.crawler.spider}, ) - assert self.crawler.stats - self.crawler.stats.inc_value("robotstxt/forbidden") + self._stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") async def robot_parser(self, request: Request) -> RobotParser | None: @@ -95,8 +96,6 @@ class RobotsTxtMiddleware: meta={"dont_obey_robotstxt": True}, callback=NO_CALLBACK, ) - assert self.crawler.engine - assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) await self._parse_robots(resp, netloc, request) @@ -109,7 +108,7 @@ class RobotsTxtMiddleware: extra={"spider": self.crawler.spider}, ) self._robots_error(e, netloc) - self.crawler.stats.inc_value("robotstxt/request_count") + self._stats.inc_value("robotstxt/request_count") parser = self._parsers[netloc] if isinstance(parser, Deferred): @@ -119,11 +118,8 @@ class RobotsTxtMiddleware: async def _parse_robots( self, response: Response, netloc: str, request: Request ) -> None: - assert self.crawler.stats - self.crawler.stats.inc_value("robotstxt/response_count") - self.crawler.stats.inc_value( - f"robotstxt/response_status_count/{response.status}" - ) + self._stats.inc_value("robotstxt/response_count") + self._stats.inc_value(f"robotstxt/response_status_count/{response.status}") rp = self._parserimpl.from_crawler(self.crawler, response.body) await self.crawler.signals.send_catch_log_async( signal=signals.robots_parsed, @@ -138,8 +134,7 @@ class RobotsTxtMiddleware: def _robots_error(self, exc: Exception, netloc: str) -> None: if not isinstance(exc, IgnoreRequest): key = f"robotstxt/exception_count/{type(exc)}" - assert self.crawler.stats - self.crawler.stats.inc_value(key) + self._stats.inc_value(key) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index bafa931de..07de1c2e7 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -43,7 +43,6 @@ class DownloaderStats: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("DOWNLOADER_STATS"): raise NotConfigured - assert crawler.stats return cls(crawler.stats) @_warn_spider_arg diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 36fb0f97d..09f0be63b 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -95,7 +95,6 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.request_fingerprinter debug = crawler.settings.getbool("DUPEFILTER_DEBUG") return cls( job_dir(crawler.settings), @@ -134,5 +133,4 @@ class RFPDupeFilter(BaseDupeFilter): self.logger.debug(msg, {"request": request}, extra={"spider": spider}) self.logdupes = False - assert spider.crawler.stats spider.crawler.stats.inc_value("dupefilter/filtered") diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 9cb792e30..5d40e5f9f 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -102,7 +102,6 @@ class CloseSpider: self._close_spider("closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: - assert self.crawler.engine self.task = call_later( self.close_on["timeout"], self._close_spider, "closespider_timeout" ) @@ -146,5 +145,4 @@ class CloseSpider: self._close_spider("closespider_timeout_no_item") def _close_spider(self, reason: str) -> None: - assert self.crawler.engine _schedule_coro(self.crawler.engine.close_spider_async(reason=reason)) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 6a5e55992..b464942af 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -26,7 +26,6 @@ class CoreStats: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 5def7509e..8802a3b58 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -45,7 +45,6 @@ class StackTraceDump: return cls(crawler) def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None: - assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e4294ac90..32abc82d3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -611,7 +611,6 @@ class FeedExporter: logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" slot_type = type(slot.storage).__name__ - assert self.crawler.stats try: await ensure_awaitable(slot.storage.store(self._get_file(slot))) except Exception: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dbb79b02d..d008d0f67 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -261,7 +261,6 @@ class DbmCacheStorage: extra={"spider": spider}, ) - assert spider.crawler.request_fingerprinter self._fingerprinter: RequestFingerprinterProtocol = ( spider.crawler.request_fingerprinter ) @@ -326,7 +325,6 @@ class FilesystemCacheStorage: extra={"spider": spider}, ) - assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter def close_spider(self, spider: Spider) -> None: diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 6c94d947e..b818569ba 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -37,7 +37,6 @@ class LogStats: interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured - assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 1fde6b296..35ff90e3b 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -29,7 +29,6 @@ class MemoryDebugger: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured - assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index e0e289ce8..ec761cfbf 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) @@ -43,6 +44,7 @@ class MemoryUsage: raise NotConfigured from exc self.crawler: Crawler = crawler + self._stats: StatsCollector = crawler.stats self.warned: bool = False self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") if self.notify_mails: # pragma: no cover @@ -77,8 +79,7 @@ class MemoryUsage: return size def engine_started(self) -> None: - assert self.crawler.stats - self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) + self._stats.set_value("memusage/startup", self.get_virtual_size()) self.tasks: list[AsyncioLoopingCall | LoopingCall] = [] tsk = create_looping_call(self.update) self.tasks.append(tsk) @@ -98,15 +99,12 @@ class MemoryUsage: tsk.stop() def update(self) -> None: - assert self.crawler.stats - self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) + self._stats.max_value("memusage/max", self.get_virtual_size()) def _check_limit(self) -> None: - assert self.crawler.engine - assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: - self.crawler.stats.set_value("memusage/limit_reached", 1) + self._stats.set_value("memusage/limit_reached", 1) mem = self.limit / 1024 / 1024 logger.error( "Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", @@ -119,7 +117,7 @@ class MemoryUsage: f"memory usage exceeded {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) - self.crawler.stats.set_value("memusage/limit_notified", 1) + self._stats.set_value("memusage/limit_notified", 1) if self.crawler.engine.spider is not None: _schedule_coro( @@ -136,9 +134,8 @@ class MemoryUsage: def _check_warning(self) -> None: if self.warned: # warn only once return - assert self.crawler.stats if self.get_virtual_size() > self.warning: - self.crawler.stats.set_value("memusage/warning_reached", 1) + self._stats.set_value("memusage/warning_reached", 1) self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached) mem = self.warning / 1024 / 1024 logger.warning( @@ -152,16 +149,13 @@ class MemoryUsage: f"memory usage reached {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) - self.crawler.stats.set_value("memusage/warning_notified", 1) + self._stats.set_value("memusage/warning_notified", 1) self.warned = True def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover """send notification mail with some additional useful info""" - assert self.crawler.engine - assert self.crawler.stats - stats = self.crawler.stats - s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" - s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" + s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" + s += f"Maximum memory usage : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n" s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n" s += ( diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index cbcc8b70e..adffbcbc4 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -87,7 +87,6 @@ class PeriodicLog: ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured - assert crawler.stats assert ext_stats is not None assert ext_delta is not None o = cls( diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index f05595806..7647cf33d 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -42,7 +42,6 @@ class StatsMailer: if not recipients: raise NotConfigured mail: MailSender = MailSender.from_crawler(crawler) - assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 1506cb1ea..392f79299 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -108,7 +108,6 @@ class TelnetConsole(protocol.ServerFactory): def _get_telnet_vars(self) -> dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - assert self.crawler.engine telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cde73f12e..f44aee03d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -45,7 +45,6 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay() self.maxdelay = self._max_delay() - assert self.crawler.engine self.crawler.engine.downloader._delay = self._start_delay() def _min_delay(self) -> float: @@ -98,7 +97,6 @@ class AutoThrottle: key: str | None = request.meta.get("download_slot") if key is None: return None, None - assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 55a3676e5..e666f4ddd 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -697,9 +697,9 @@ class FilesPipeline(MediaPipeline): } def inc_stats(self, status: str) -> None: - assert self.crawler.stats - self.crawler.stats.inc_value("file_count") - self.crawler.stats.inc_value(f"file_status_count/{status}") + stats = self.crawler.stats + stats.inc_value("file_count") + stats.inc_value(f"file_status_count/{status}") async def _file_downloaded( self, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 764f82a78..d4025bf3b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -100,7 +100,6 @@ class MediaPipeline(ABC): stacklevel=2, ) self.crawler: Crawler = crawler - assert crawler.request_fingerprinter self._fingerprinter: RequestFingerprinterProtocol = ( crawler.request_fingerprinter ) @@ -228,7 +227,6 @@ class MediaPipeline(ABC): ) -> FileInfo: try: self._modify_media_request(request) - assert self.crawler.engine response = await self.crawler.engine.download_async(request) return await ensure_awaitable( self.media_downloaded(response, request, info, item=item) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 8e9783f5a..f4c35c0fe 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -265,7 +265,6 @@ class ScrapyPriorityQueue: class DownloaderInterface: def __init__(self, crawler: Crawler): - assert crawler.engine self.downloader: Downloader = crawler.engine.downloader def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: diff --git a/scrapy/shell.py b/scrapy/shell.py index dfea00c46..8f78af571 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -193,7 +193,6 @@ class Shell: """ if not self.spider: await self._open_spider(spider) - assert self.crawler.engine is not None # send the request to the engine self.crawler.engine.crawl(request) # this will fire when the request callback runs (via the callback hijacking in _request_deferred()) @@ -204,7 +203,6 @@ class Shell: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider - assert self.crawler.engine await self.crawler.engine.open_spider_async(close_if_idle=False) self.spider = spider diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1781543e3..49683168e 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -70,7 +70,6 @@ class DepthMiddleware(BaseSpiderMiddleware): maxdepth = settings.getint("DEPTH_LIMIT") verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") - assert crawler.stats o = cls(maxdepth, crawler.stats, verbose, prio) o.crawler = crawler return o diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 156b73e7e..116a02733 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -78,9 +78,9 @@ class HttpErrorMiddleware: self, response: Response, exception: Exception, spider: Spider | None = None ) -> Iterable[Any] | None: if isinstance(exception, HttpError): - assert self.crawler.stats - self.crawler.stats.inc_value("httperror/response_ignored_count") - self.crawler.stats.inc_value( + stats = self.crawler.stats + stats.inc_value("httperror/response_ignored_count") + stats.inc_value( f"httperror/response_ignored_status_count/{response.status}" ) logger.info( diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index f325ce7a0..86bdd2ed6 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -48,6 +48,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware): {"maxlength": self.maxlength, "url": request.url}, extra={"spider": self.crawler.spider}, ) - assert self.crawler.stats self.crawler.stats.inc_value("urllength/request_ignored_count") return None diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index ee357c031..bfa39169f 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -239,7 +239,6 @@ class LogCounterHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" - assert self.crawler.stats self.crawler.stats.inc_value(sname) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 358f20ed7..17ca02dea 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -74,6 +74,31 @@ class TestCrawler: assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + "attr", + ["extensions", "logformatter", "request_fingerprinter", "stats"], + ) + def test_late_attr_before_apply_settings(self, attr: str) -> None: + crawler = get_raw_crawler(DefaultSpider) + with pytest.raises(RuntimeError, match=rf"Crawler\.{attr} is not set yet"): + getattr(crawler, attr) + crawler._apply_settings() + assert getattr(crawler, attr) is not None + + @pytest.mark.parametrize( + "attr", + ["engine", "extensions", "logformatter", "request_fingerprinter", "stats"], + ) + def test_late_attr_on_class(self, attr: str) -> None: + # Introspection tools such as help() read these off the class. + assert getattr(Crawler, attr) is getattr(Crawler, attr) + + def test_late_attr_engine_before_crawl(self) -> None: + crawler = get_raw_crawler(DefaultSpider) + crawler._apply_settings() + with pytest.raises(RuntimeError, match=r"Crawler\.engine is not set yet"): + _ = crawler.engine + @pytest.mark.parametrize( ("attr", "setting"), [