RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Type late Crawler attributes as always set instead of None (#7882)

This commit is contained in:
Adrian 2026-08-10 09:07:34 +02:00 committed by GitHub
parent 09c918115b
commit 508bd7faec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
32 changed files with 131 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/topics/api.rst
View File

@ -35,6 +35,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`,
:attr:`request_fingerprinter` and :attr:`stats` attributes get their value
when the crawl starts, and raise :exc:`RuntimeError` when read before that.
.. versionchanged:: VERSION
Those attributes used to be ``None`` before getting their value.
.. attribute:: request_fingerprinter
The request fingerprint builder of this crawler.

1
scrapy/commands/parse.py
View File

@ -281,7 +281,6 @@ class Command(BaseRunSpiderCommand):
) -> list[Any]:
items, requests, opts, depth, spider, callback = args
if opts.pipelines:
assert self.pcrawler.engine
itemproc = self.pcrawler.engine.scraper.itemproc
if hasattr(itemproc, "process_item_async"):
for item in items:

24
scrapy/core/engine.py
View File

@ -112,7 +112,6 @@ class ExecutionEngine:
self.crawler: Crawler = crawler
self.settings: Settings = crawler.settings
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
self._slot: _Slot | None = None
self.spider: Spider | None = None
@ -299,7 +298,6 @@ class ExecutionEngine:
response=None,
spider=self.spider,
)
assert self.crawler.stats
self.crawler.stats.inc_value("spider_exceptions/count")
self.crawler.stats.inc_value(
f"spider_exceptions/{type(exception).__name__}"
@ -563,17 +561,17 @@ class ExecutionEngine:
if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
await maybe_deferred_to_future(d)
await self.scraper.open_spider_async()
assert self.crawler.stats
if argument_is_required(self.crawler.stats.open_spider, "spider"):
stats = self.crawler.stats
if argument_is_required(stats.open_spider, "spider"):
warnings.warn(
f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument,"
f" this is deprecated and the argument will not be passed in future Scrapy versions.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self.crawler.stats.open_spider(spider=self.crawler.spider)
stats.open_spider(spider=self.crawler.spider)
else:
self.crawler.stats.open_spider()
stats.open_spider()
await self.signals.send_catch_log_async(
signals.spider_opened, spider=self.crawler.spider
)
@ -676,20 +674,18 @@ class ExecutionEngine:
extra={"spider": spider},
)
assert self.crawler.stats
try:
if argument_is_required(self.crawler.stats.close_spider, "spider"):
stats = self.crawler.stats
if argument_is_required(stats.close_spider, "spider"):
warnings.warn(
f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
f"The close_spider() method of {global_object_name(type(stats))} requires a spider argument,"
f" this is deprecated and the argument will not be passed in future Scrapy versions.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self.crawler.stats.close_spider(
spider=self.crawler.spider, reason=reason
)
stats.close_spider(spider=self.crawler.spider, reason=reason)
else:
self.crawler.stats.close_spider(reason=reason)
stats.close_spider(reason=reason)
except Exception:
logger.error("Stats close failure")

11
scrapy/core/scraper.py
View File

@ -120,7 +120,6 @@ class Scraper:
self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
self.crawler: Crawler = crawler
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
def _check_deprecated_itemproc_method(self, method: str) -> None:
@ -355,7 +354,6 @@ class Scraper:
assert self.crawler.spider
exc = _failure.value
if isinstance(exc, CloseSpider):
assert self.crawler.engine is not None # typing
_schedule_coro(
self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")
)
@ -374,11 +372,9 @@ class Scraper:
response=response,
spider=self.crawler.spider,
)
assert self.crawler.stats
self.crawler.stats.inc_value("spider_exceptions/count")
self.crawler.stats.inc_value(
f"spider_exceptions/{_failure.value.__class__.__name__}"
)
stats = self.crawler.stats
stats.inc_value("spider_exceptions/count")
stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}")
def handle_spider_output(
self,
@ -456,7 +452,6 @@ class Scraper:
Items are sent to the item pipelines, requests are scheduled.
"""
if isinstance(output, Request):
assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
return
if output is not None:

78
scrapy/crawler.py
View File

@ -8,7 +8,7 @@ import signal
import warnings
from abc import ABC, abstractmethod
from functools import partial
from typing import TYPE_CHECKING, Any, TypeVar
from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload
from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
@ -58,7 +58,55 @@ logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class _LateAttribute(Generic[_T]):
"""Descriptor for a :class:`Crawler` attribute that only gets a value once
the crawl starts.
The value is kept in an attribute of the same name prefixed with an
underscore, and reading it before it is set raises :exc:`RuntimeError`.
This way the public attribute can be annotated as always set, and its
users, both in Scrapy and in third-party code, do not need to narrow its
type on every use. Code that runs before the crawl starts reads the
underscore-prefixed attribute instead.
"""
def __set_name__(self, owner: type[Crawler], name: str) -> None:
self._name = name
self._private_name = f"_{name}"
@overload
def __get__(self, instance: None, owner: type[Crawler]) -> _LateAttribute[_T]: ...
@overload
def __get__(self, instance: Crawler, owner: type[Crawler]) -> _T: ...
def __get__(
self, instance: Crawler | None, owner: type[Crawler]
) -> _LateAttribute[_T] | _T:
if instance is None:
return self
value: _T | None = getattr(instance, self._private_name)
if value is None:
raise RuntimeError(
f"Crawler.{self._name} is not set yet. It is set when the "
"crawl starts, so it can only be used from then on, e.g. "
"from the spider_opened signal handler onwards."
)
return value
def __set__(self, instance: Crawler, value: _T) -> None:
setattr(instance, self._private_name, value)
class Crawler:
engine: _LateAttribute[ExecutionEngine] = _LateAttribute()
extensions: _LateAttribute[ExtensionManager] = _LateAttribute()
logformatter: _LateAttribute[LogFormatter] = _LateAttribute()
request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = (
_LateAttribute()
)
stats: _LateAttribute[StatsCollector] = _LateAttribute()
def __init__(
self,
spidercls: type[Spider],
@ -83,12 +131,13 @@ class Crawler:
self.crawling: bool = False
self._started: bool = False
self.extensions: ExtensionManager | None = None
self.stats: StatsCollector | None = None
self.logformatter: LogFormatter | None = None
self.request_fingerprinter: RequestFingerprinterProtocol | None = None
self.spider: Spider | None = None
self.engine: ExecutionEngine | None = None
self._engine: ExecutionEngine | None = None
self._extensions: ExtensionManager | None = None
self._logformatter: LogFormatter | None = None
self._request_fingerprinter: RequestFingerprinterProtocol | None = None
self._stats: StatsCollector | None = None
def _update_root_log_handler(self) -> None:
if get_scrapy_root_handler() is not None:
@ -225,8 +274,8 @@ class Crawler:
yield deferred_from_coro(self.engine.start_async())
except Exception:
self.crawling = False
if self.engine is not None:
yield deferred_from_coro(self.engine.close_async())
if self._engine is not None:
yield deferred_from_coro(self._engine.close_async())
raise
async def crawl_async(self, *args: Any, **kwargs: Any) -> None:
@ -255,8 +304,8 @@ class Crawler:
await self.engine.start_async()
except Exception:
self.crawling = False
if self.engine is not None:
await self.engine.close_async()
if self._engine is not None:
await self._engine.close_async()
raise
def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
@ -282,7 +331,6 @@ class Crawler:
"""
if self.crawling:
self.crawling = False
assert self.engine
if self.engine.running:
await self.engine.stop_async()
@ -313,7 +361,7 @@ class Crawler:
This method can only be called after the crawl engine has been created,
e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
"""
if not self.engine:
if self._engine is None:
raise RuntimeError(
"Crawler.get_downloader_middleware() can only be called after "
"the crawl engine has been created."
@ -331,7 +379,7 @@ class Crawler:
created, e.g. at signals :signal:`engine_started` or
:signal:`spider_opened`.
"""
if not self.extensions:
if self._extensions is None:
raise RuntimeError(
"Crawler.get_extension() can only be called after the "
"extension manager has been created."
@ -348,7 +396,7 @@ class Crawler:
This method can only be called after the crawl engine has been created,
e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
"""
if not self.engine:
if self._engine is None:
raise RuntimeError(
"Crawler.get_item_pipeline() can only be called after the "
"crawl engine has been created."
@ -365,7 +413,7 @@ class Crawler:
This method can only be called after the crawl engine has been created,
e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
"""
if not self.engine:
if self._engine is None:
raise RuntimeError(
"Crawler.get_spider_middleware() can only be called after the "
"crawl engine has been created."

1
scrapy/downloadermiddlewares/httpcache.py
View File

@ -55,7 +55,6 @@ class HttpCacheMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)

1
scrapy/downloadermiddlewares/offsite.py
View File

@ -61,7 +61,6 @@ class OffsiteMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)

1
scrapy/downloadermiddlewares/retry.py
View File

@ -94,7 +94,6 @@ def get_retry_request(
retry-related job stats
"""
settings = spider.crawler.settings
assert spider.crawler.stats
stats = spider.crawler.stats
retry_times = request.meta.get("retry_times", 0) + 1
if max_retry_times is None:

19
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -27,6 +27,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.robotstxt import RobotParser
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
@ -41,6 +42,7 @@ class RobotsTxtMiddleware:
self._default_useragent: str = crawler.settings["USER_AGENT"]
self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
self.crawler: Crawler = crawler
self._stats: StatsCollector = crawler.stats
self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}
self._parserimpl: RobotParser = load_object(
crawler.settings.get("ROBOTSTXT_PARSER")
@ -78,8 +80,7 @@ class RobotsTxtMiddleware:
{"request": request},
extra={"spider": self.crawler.spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden")
self._stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
async def robot_parser(self, request: Request) -> RobotParser | None:
@ -95,8 +96,6 @@ class RobotsTxtMiddleware:
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
assert self.crawler.engine
assert self.crawler.stats
try:
resp = await self.crawler.engine.download_async(robotsreq)
await self._parse_robots(resp, netloc, request)
@ -109,7 +108,7 @@ class RobotsTxtMiddleware:
extra={"spider": self.crawler.spider},
)
self._robots_error(e, netloc)
self.crawler.stats.inc_value("robotstxt/request_count")
self._stats.inc_value("robotstxt/request_count")
parser = self._parsers[netloc]
if isinstance(parser, Deferred):
@ -119,11 +118,8 @@ class RobotsTxtMiddleware:
async def _parse_robots(
self, response: Response, netloc: str, request: Request
) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
self._stats.inc_value("robotstxt/response_count")
self._stats.inc_value(f"robotstxt/response_status_count/{response.status}")
rp = self._parserimpl.from_crawler(self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
@ -138,8 +134,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, exc: Exception, netloc: str) -> None:
if not isinstance(exc, IgnoreRequest):
key = f"robotstxt/exception_count/{type(exc)}"
assert self.crawler.stats
self.crawler.stats.inc_value(key)
self._stats.inc_value(key)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = None

1
scrapy/downloadermiddlewares/stats.py
View File

@ -43,7 +43,6 @@ class DownloaderStats:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("DOWNLOADER_STATS"):
raise NotConfigured
assert crawler.stats
return cls(crawler.stats)
@_warn_spider_arg

2
scrapy/dupefilters.py
View File

@ -95,7 +95,6 @@ class RFPDupeFilter(BaseDupeFilter):
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.request_fingerprinter
debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
return cls(
job_dir(crawler.settings),
@ -134,5 +133,4 @@ class RFPDupeFilter(BaseDupeFilter):
self.logger.debug(msg, {"request": request}, extra={"spider": spider})
self.logdupes = False
assert spider.crawler.stats
spider.crawler.stats.inc_value("dupefilter/filtered")

2
scrapy/extensions/closespider.py
View File

@ -102,7 +102,6 @@ class CloseSpider:
self._close_spider("closespider_pagecount_no_item")
def spider_opened(self, spider: Spider) -> None:
assert self.crawler.engine
self.task = call_later(
self.close_on["timeout"], self._close_spider, "closespider_timeout"
)
@ -146,5 +145,4 @@ class CloseSpider:
self._close_spider("closespider_timeout_no_item")
def _close_spider(self, reason: str) -> None:
assert self.crawler.engine
_schedule_coro(self.crawler.engine.close_spider_async(reason=reason))

1
scrapy/extensions/corestats.py
View File

@ -26,7 +26,6 @@ class CoreStats:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)

1
scrapy/extensions/debug.py
View File

@ -45,7 +45,6 @@ class StackTraceDump:
return cls(crawler)
def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None:
assert self.crawler.engine
log_args = {
"stackdumps": self._thread_stacks(),
"enginestatus": format_engine_status(self.crawler.engine),

1
scrapy/extensions/feedexport.py
View File

@ -611,7 +611,6 @@ class FeedExporter:
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
slot_type = type(slot.storage).__name__
assert self.crawler.stats
try:
await ensure_awaitable(slot.storage.store(self._get_file(slot)))
except Exception:

2
scrapy/extensions/httpcache.py
View File

@ -261,7 +261,6 @@ class DbmCacheStorage:
extra={"spider": spider},
)
assert spider.crawler.request_fingerprinter
self._fingerprinter: RequestFingerprinterProtocol = (
spider.crawler.request_fingerprinter
)
@ -326,7 +325,6 @@ class FilesystemCacheStorage:
extra={"spider": spider},
)
assert spider.crawler.request_fingerprinter
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider: Spider) -> None:

1
scrapy/extensions/logstats.py
View File

@ -37,7 +37,6 @@ class LogStats:
interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
if not interval:
raise NotConfigured
assert crawler.stats
o = cls(crawler.stats, interval)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)

1
scrapy/extensions/memdebug.py
View File

@ -29,7 +29,6 @@ class MemoryDebugger:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
raise NotConfigured
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o

26
scrapy/extensions/memusage.py
View File

@ -27,6 +27,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
@ -43,6 +44,7 @@ class MemoryUsage:
raise NotConfigured from exc
self.crawler: Crawler = crawler
self._stats: StatsCollector = crawler.stats
self.warned: bool = False
self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
if self.notify_mails: # pragma: no cover
@ -77,8 +79,7 @@ class MemoryUsage:
return size
def engine_started(self) -> None:
assert self.crawler.stats
self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
self._stats.set_value("memusage/startup", self.get_virtual_size())
self.tasks: list[AsyncioLoopingCall | LoopingCall] = []
tsk = create_looping_call(self.update)
self.tasks.append(tsk)
@ -98,15 +99,12 @@ class MemoryUsage:
tsk.stop()
def update(self) -> None:
assert self.crawler.stats
self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
self._stats.max_value("memusage/max", self.get_virtual_size())
def _check_limit(self) -> None:
assert self.crawler.engine
assert self.crawler.stats
peak_mem_usage = self.get_virtual_size()
if peak_mem_usage > self.limit:
self.crawler.stats.set_value("memusage/limit_reached", 1)
self._stats.set_value("memusage/limit_reached", 1)
mem = self.limit / 1024 / 1024
logger.error(
"Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
@ -119,7 +117,7 @@ class MemoryUsage:
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value("memusage/limit_notified", 1)
self._stats.set_value("memusage/limit_notified", 1)
if self.crawler.engine.spider is not None:
_schedule_coro(
@ -136,9 +134,8 @@ class MemoryUsage:
def _check_warning(self) -> None:
if self.warned: # warn only once
return
assert self.crawler.stats
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value("memusage/warning_reached", 1)
self._stats.set_value("memusage/warning_reached", 1)
self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached)
mem = self.warning / 1024 / 1024
logger.warning(
@ -152,16 +149,13 @@ class MemoryUsage:
f"memory usage reached {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value("memusage/warning_notified", 1)
self._stats.set_value("memusage/warning_notified", 1)
self.warned = True
def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover
"""send notification mail with some additional useful info"""
assert self.crawler.engine
assert self.crawler.stats
stats = self.crawler.stats
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
s += f"Maximum memory usage : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"
s += (

1
scrapy/extensions/periodic_log.py
View File

@ -87,7 +87,6 @@ class PeriodicLog:
)
if not (ext_stats or ext_delta or ext_timing_enabled):
raise NotConfigured
assert crawler.stats
assert ext_stats is not None
assert ext_delta is not None
o = cls(

1
scrapy/extensions/statsmailer.py
View File

@ -42,7 +42,6 @@ class StatsMailer:
if not recipients:
raise NotConfigured
mail: MailSender = MailSender.from_crawler(crawler)
assert crawler.stats
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o

1
scrapy/extensions/telnet.py
View File

@ -108,7 +108,6 @@ class TelnetConsole(protocol.ServerFactory):
def _get_telnet_vars(self) -> dict[str, Any]:
# Note: if you add entries here also update topics/telnetconsole.rst
assert self.crawler.engine
telnet_vars: dict[str, Any] = {
"engine": self.crawler.engine,
"spider": self.crawler.engine.spider,

2
scrapy/extensions/throttle.py
View File

@ -45,7 +45,6 @@ class AutoThrottle:
def _spider_opened(self, spider: Spider) -> None:
self.mindelay = self._min_delay()
self.maxdelay = self._max_delay()
assert self.crawler.engine
self.crawler.engine.downloader._delay = self._start_delay()
def _min_delay(self) -> float:
@ -98,7 +97,6 @@ class AutoThrottle:
key: str | None = request.meta.get("download_slot")
if key is None:
return None, None
assert self.crawler.engine
return key, self.crawler.engine.downloader.slots.get(key)
def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:

6
scrapy/pipelines/files.py
View File

@ -697,9 +697,9 @@ class FilesPipeline(MediaPipeline):
}
def inc_stats(self, status: str) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("file_count")
self.crawler.stats.inc_value(f"file_status_count/{status}")
stats = self.crawler.stats
stats.inc_value("file_count")
stats.inc_value(f"file_status_count/{status}")
async def _file_downloaded(
self,

2
scrapy/pipelines/media.py
View File

@ -100,7 +100,6 @@ class MediaPipeline(ABC):
stacklevel=2,
)
self.crawler: Crawler = crawler
assert crawler.request_fingerprinter
self._fingerprinter: RequestFingerprinterProtocol = (
crawler.request_fingerprinter
)
@ -228,7 +227,6 @@ class MediaPipeline(ABC):
) -> FileInfo:
try:
self._modify_media_request(request)
assert self.crawler.engine
response = await self.crawler.engine.download_async(request)
return await ensure_awaitable(
self.media_downloaded(response, request, info, item=item)

1
scrapy/pqueues.py
View File

@ -265,7 +265,6 @@ class ScrapyPriorityQueue:
class DownloaderInterface:
def __init__(self, crawler: Crawler):
assert crawler.engine
self.downloader: Downloader = crawler.engine.downloader
def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]:

2
scrapy/shell.py
View File

@ -193,7 +193,6 @@ class Shell:
"""
if not self.spider:
await self._open_spider(spider)
assert self.crawler.engine is not None
# send the request to the engine
self.crawler.engine.crawl(request)
# this will fire when the request callback runs (via the callback hijacking in _request_deferred())
@ -204,7 +203,6 @@ class Shell:
spider = self.crawler.spider or self.crawler._create_spider()
self.crawler.spider = spider
assert self.crawler.engine
await self.crawler.engine.open_spider_async(close_if_idle=False)
self.spider = spider

1
scrapy/spidermiddlewares/depth.py
View File

@ -70,7 +70,6 @@ class DepthMiddleware(BaseSpiderMiddleware):
maxdepth = settings.getint("DEPTH_LIMIT")
verbose = settings.getbool("DEPTH_STATS_VERBOSE")
prio = settings.getint("DEPTH_PRIORITY")
assert crawler.stats
o = cls(maxdepth, crawler.stats, verbose, prio)
o.crawler = crawler
return o

6
scrapy/spidermiddlewares/httperror.py
View File

@ -78,9 +78,9 @@ class HttpErrorMiddleware:
self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None:
if isinstance(exception, HttpError):
assert self.crawler.stats
self.crawler.stats.inc_value("httperror/response_ignored_count")
self.crawler.stats.inc_value(
stats = self.crawler.stats
stats.inc_value("httperror/response_ignored_count")
stats.inc_value(
f"httperror/response_ignored_status_count/{response.status}"
)
logger.info(

1
scrapy/spidermiddlewares/urllength.py
View File

@ -48,6 +48,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware):
{"maxlength": self.maxlength, "url": request.url},
extra={"spider": self.crawler.spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value("urllength/request_ignored_count")
return None

1
scrapy/utils/log.py
View File

@ -239,7 +239,6 @@ class LogCounterHandler(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
sname = f"log_count/{record.levelname}"
assert self.crawler.stats
self.crawler.stats.inc_value(sname)

25
tests/test_crawler.py
View File

@ -74,6 +74,31 @@ class TestCrawler:
assert not settings.frozen
assert crawler.settings.frozen
@pytest.mark.parametrize(
"attr",
["extensions", "logformatter", "request_fingerprinter", "stats"],
)
def test_late_attr_before_apply_settings(self, attr: str) -> None:
crawler = get_raw_crawler(DefaultSpider)
with pytest.raises(RuntimeError, match=rf"Crawler\.{attr} is not set yet"):
getattr(crawler, attr)
crawler._apply_settings()
assert getattr(crawler, attr) is not None
@pytest.mark.parametrize(
"attr",
["engine", "extensions", "logformatter", "request_fingerprinter", "stats"],
)
def test_late_attr_on_class(self, attr: str) -> None:
# Introspection tools such as help() read these off the class.
assert getattr(Crawler, attr) is getattr(Crawler, attr)
def test_late_attr_engine_before_crawl(self) -> None:
crawler = get_raw_crawler(DefaultSpider)
crawler._apply_settings()
with pytest.raises(RuntimeError, match=r"Crawler\.engine is not set yet"):
_ = crawler.engine
@pytest.mark.parametrize(
("attr", "setting"),
[