RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate spider arguments of StatsCollector methods (#7011)

This commit is contained in:
Andrey Rakhmatullin 2025-08-20 12:52:22 +05:00 committed by GitHub
parent edba1ad572
commit b68330811b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 141 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/topics/api.rst
View File

@ -280,13 +280,13 @@ class (which they all inherit from).
The following methods are not part of the stats collection api but instead
used when implementing custom stats collectors:
.. method:: open_spider(spider)
.. method:: open_spider()
Open the given spider for stats collection.
Open the spider for stats collection.
.. method:: close_spider(spider)
.. method:: close_spider()
Close the given spider. After this is called, no more specific stats
Close the spider. After this is called, no more specific stats
can be accessed or collected.
Engine API

4
scrapy/core/engine.py
View File

@ -519,7 +519,7 @@ class ExecutionEngine:
await maybe_deferred_to_future(d)
await self.scraper.open_spider_async()
assert self.crawler.stats
self.crawler.stats.open_spider(self.crawler.spider)
self.crawler.stats.open_spider()
await self.signals.send_catch_log_async(
signals.spider_opened, spider=self.crawler.spider
)
@ -614,7 +614,7 @@ class ExecutionEngine:
assert self.crawler.stats
try:
self.crawler.stats.close_spider(spider, reason=reason)
self.crawler.stats.close_spider(reason=reason)
except Exception:
log_failure("Stats close failure")

14
scrapy/core/scheduler.py
View File

@ -348,11 +348,11 @@ class Scheduler(BaseScheduler):
dqok = self._dqpush(request)
assert self.stats is not None
if dqok:
self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
self.stats.inc_value("scheduler/enqueued/disk")
else:
self._mqpush(request)
self.stats.inc_value("scheduler/enqueued/memory", spider=self.spider)
self.stats.inc_value("scheduler/enqueued", spider=self.spider)
self.stats.inc_value("scheduler/enqueued/memory")
self.stats.inc_value("scheduler/enqueued")
return True
def next_request(self) -> Request | None:
@ -367,13 +367,13 @@ class Scheduler(BaseScheduler):
request: Request | None = self.mqs.pop()
assert self.stats is not None
if request is not None:
self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
self.stats.inc_value("scheduler/dequeued/memory")
else:
request = self._dqpop()
if request is not None:
self.stats.inc_value("scheduler/dequeued/disk", spider=self.spider)
self.stats.inc_value("scheduler/dequeued/disk")
if request is not None:
self.stats.inc_value("scheduler/dequeued", spider=self.spider)
self.stats.inc_value("scheduler/dequeued")
return request
def __len__(self) -> int:
@ -402,7 +402,7 @@ class Scheduler(BaseScheduler):
)
self.logunser = False
assert self.stats is not None
self.stats.inc_value("scheduler/unserializable", spider=self.spider)
self.stats.inc_value("scheduler/unserializable")
return False
return True

7
scrapy/core/scraper.py
View File

@ -381,12 +381,9 @@ class Scraper:
spider=self.crawler.spider,
)
assert self.crawler.stats
self.crawler.stats.inc_value("spider_exceptions/count")
self.crawler.stats.inc_value(
"spider_exceptions/count", spider=self.crawler.spider
)
self.crawler.stats.inc_value(
f"spider_exceptions/{_failure.value.__class__.__name__}",
spider=self.crawler.spider,
f"spider_exceptions/{_failure.value.__class__.__name__}"
)
def handle_spider_output(

18
scrapy/downloadermiddlewares/httpcache.py
View File

@ -83,16 +83,16 @@ class HttpCacheMiddleware:
spider, request
)
if cachedresponse is None:
self.stats.inc_value("httpcache/miss", spider=spider)
self.stats.inc_value("httpcache/miss")
if self.ignore_missing:
self.stats.inc_value("httpcache/ignore", spider=spider)
self.stats.inc_value("httpcache/ignore")
raise IgnoreRequest(f"Ignored request not in cache: {request}")
return None # first time request
# Return cached response only if not expired
cachedresponse.flags.append("cached")
if self.policy.is_cached_response_fresh(cachedresponse, request):
self.stats.inc_value("httpcache/hit", spider=spider)
self.stats.inc_value("httpcache/hit")
return cachedresponse
# Keep a reference to cached response to avoid a second cache lookup on
@ -120,15 +120,15 @@ class HttpCacheMiddleware:
# Do not validate first-hand responses
cachedresponse: Response | None = request.meta.pop("cached_response", None)
if cachedresponse is None:
self.stats.inc_value("httpcache/firsthand", spider=spider)
self.stats.inc_value("httpcache/firsthand")
self._cache_response(spider, response, request, cachedresponse)
return response
if self.policy.is_cached_response_valid(cachedresponse, response, request):
self.stats.inc_value("httpcache/revalidate", spider=spider)
self.stats.inc_value("httpcache/revalidate")
return cachedresponse
self.stats.inc_value("httpcache/invalidate", spider=spider)
self.stats.inc_value("httpcache/invalidate")
self._cache_response(spider, response, request, cachedresponse)
return response
@ -139,7 +139,7 @@ class HttpCacheMiddleware:
if cachedresponse is not None and isinstance(
exception, self.DOWNLOAD_EXCEPTIONS
):
self.stats.inc_value("httpcache/errorrecovery", spider=spider)
self.stats.inc_value("httpcache/errorrecovery")
return cachedresponse
return None
@ -151,7 +151,7 @@ class HttpCacheMiddleware:
cachedresponse: Response | None,
) -> None:
if self.policy.should_cache_response(response, request):
self.stats.inc_value("httpcache/store", spider=spider)
self.stats.inc_value("httpcache/store")
self.storage.store_response(spider, request, response)
else:
self.stats.inc_value("httpcache/uncacheable", spider=spider)
self.stats.inc_value("httpcache/uncacheable")

5
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -133,11 +133,8 @@ class HttpCompressionMiddleware:
self.stats.inc_value(
"httpcompression/response_bytes",
len(decoded_body),
spider=spider,
)
self.stats.inc_value(
"httpcompression/response_count", spider=spider
)
self.stats.inc_value("httpcompression/response_count")
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)

4
scrapy/downloadermiddlewares/offsite.py
View File

@ -54,8 +54,8 @@ class OffsiteMiddleware:
{"domain": domain, "request": request},
extra={"spider": spider},
)
self.stats.inc_value("offsite/domains", spider=spider)
self.stats.inc_value("offsite/filtered", spider=spider)
self.stats.inc_value("offsite/domains")
self.stats.inc_value("offsite/filtered")
raise IgnoreRequest
def should_follow(self, request: Request, spider: Spider) -> bool:

22
scrapy/downloadermiddlewares/stats.py
View File

@ -48,21 +48,17 @@ class DownloaderStats:
def process_request(
self, request: Request, spider: Spider
) -> Request | Response | None:
self.stats.inc_value("downloader/request_count", spider=spider)
self.stats.inc_value(
f"downloader/request_method_count/{request.method}", spider=spider
)
self.stats.inc_value("downloader/request_count")
self.stats.inc_value(f"downloader/request_method_count/{request.method}")
reqlen = len(request_httprepr(request))
self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
self.stats.inc_value("downloader/request_bytes", reqlen)
return None
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Request | Response:
self.stats.inc_value("downloader/response_count", spider=spider)
self.stats.inc_value(
f"downloader/response_status_count/{response.status}", spider=spider
)
self.stats.inc_value("downloader/response_count")
self.stats.inc_value(f"downloader/response_status_count/{response.status}")
reslen = (
len(response.body)
+ get_header_size(response.headers)
@ -70,15 +66,13 @@ class DownloaderStats:
+ 4
)
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
self.stats.inc_value("downloader/response_bytes", reslen)
return response
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Request | Response | None:
ex_class = global_object_name(exception.__class__)
self.stats.inc_value("downloader/exception_count", spider=spider)
self.stats.inc_value(
f"downloader/exception_type_count/{ex_class}", spider=spider
)
self.stats.inc_value("downloader/exception_count")
self.stats.inc_value(f"downloader/exception_type_count/{ex_class}")
return None

2
scrapy/dupefilters.py
View File

@ -152,4 +152,4 @@ class RFPDupeFilter(BaseDupeFilter):
self.logdupes = False
assert spider.crawler.stats
spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider)
spider.crawler.stats.inc_value("dupefilter/filtered")

18
scrapy/extensions/corestats.py
View File

@ -35,26 +35,24 @@ class CoreStats:
def spider_opened(self, spider: Spider) -> None:
self.start_time = datetime.now(tz=timezone.utc)
self.stats.set_value("start_time", self.start_time, spider=spider)
self.stats.set_value("start_time", self.start_time)
def spider_closed(self, spider: Spider, reason: str) -> None:
assert self.start_time is not None
finish_time = datetime.now(tz=timezone.utc)
elapsed_time = finish_time - self.start_time
elapsed_time_seconds = elapsed_time.total_seconds()
self.stats.set_value(
"elapsed_time_seconds", elapsed_time_seconds, spider=spider
)
self.stats.set_value("finish_time", finish_time, spider=spider)
self.stats.set_value("finish_reason", reason, spider=spider)
self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds)
self.stats.set_value("finish_time", finish_time)
self.stats.set_value("finish_reason", reason)
def item_scraped(self, item: Any, spider: Spider) -> None:
self.stats.inc_value("item_scraped_count", spider=spider)
self.stats.inc_value("item_scraped_count")
def response_received(self, spider: Spider) -> None:
self.stats.inc_value("response_received_count", spider=spider)
self.stats.inc_value("response_received_count")
def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None:
reason = exception.__class__.__name__
self.stats.inc_value("item_dropped_count", spider=spider)
self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)
self.stats.inc_value("item_dropped_count")
self.stats.inc_value(f"item_dropped_reasons_count/{reason}")

8
scrapy/extensions/memdebug.py
View File

@ -36,12 +36,8 @@ class MemoryDebugger:
def spider_closed(self, spider: Spider, reason: str) -> None:
gc.collect()
self.stats.set_value(
"memdebug/gc_garbage_count", len(gc.garbage), spider=spider
)
self.stats.set_value("memdebug/gc_garbage_count", len(gc.garbage))
for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value(
f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider
)
self.stats.set_value(f"memdebug/live_refs/{cls.__name__}", len(wdict))

12
scrapy/pipelines/files.py
View File

@ -585,7 +585,7 @@ class FilesPipeline(MediaPipeline):
{"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
extra={"spider": info.spider},
)
self.inc_stats(info.spider, "uptodate")
self.inc_stats("uptodate")
checksum = result.get("checksum", None)
return {
@ -663,7 +663,7 @@ class FilesPipeline(MediaPipeline):
{"status": status, "request": request, "referer": referer},
extra={"spider": info.spider},
)
self.inc_stats(info.spider, status)
self.inc_stats(status)
try:
path = self.file_path(request, response=response, info=info, item=item)
@ -694,10 +694,10 @@ class FilesPipeline(MediaPipeline):
"status": status,
}
def inc_stats(self, spider: Spider, status: str) -> None:
assert spider.crawler.stats
spider.crawler.stats.inc_value("file_count", spider=spider)
spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider)
def inc_stats(self, status: str) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("file_count")
self.crawler.stats.inc_value(f"file_status_count/{status}")
# Overridable Interface
def get_media_requests(

14
scrapy/spidermiddlewares/depth.py
View File

@ -55,22 +55,22 @@ class DepthMiddleware(BaseSpiderMiddleware):
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
) -> Iterable[Any]:
self._init_depth(response, spider)
self._init_depth(response)
yield from super().process_spider_output(response, result, spider)
async def process_spider_output_async(
self, response: Response, result: AsyncIterator[Any], spider: Spider
) -> AsyncIterator[Any]:
self._init_depth(response, spider)
self._init_depth(response)
async for o in super().process_spider_output_async(response, result, spider):
yield o
def _init_depth(self, response: Response, spider: Spider) -> None:
def _init_depth(self, response: Response) -> None:
# base case (depth=0)
if "depth" not in response.meta:
response.meta["depth"] = 0
if self.verbose_stats:
self.stats.inc_value("request_depth_count/0", spider=spider)
self.stats.inc_value("request_depth_count/0")
def get_processed_request(
self, request: Request, response: Response | None
@ -90,8 +90,6 @@ class DepthMiddleware(BaseSpiderMiddleware):
)
return None
if self.verbose_stats:
self.stats.inc_value(
f"request_depth_count/{depth}", spider=self.crawler.spider
)
self.stats.max_value("request_depth_max", depth, spider=self.crawler.spider)
self.stats.inc_value(f"request_depth_count/{depth}")
self.stats.max_value("request_depth_max", depth)
return request

4
scrapy/spidermiddlewares/urllength.py
View File

@ -49,7 +49,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware):
extra={"spider": self.crawler.spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value(
"urllength/request_ignored_count", spider=self.crawler.spider
)
self.crawler.stats.inc_value("urllength/request_ignored_count")
return None

56
scrapy/statscollectors.py
View File

@ -4,10 +4,14 @@ Scrapy extension for collecting scraping stats
from __future__ import annotations
import inspect
import logging
import pprint
import warnings
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
@ -23,6 +27,41 @@ class StatsCollector:
def __init__(self, crawler: Crawler):
self._dump: bool = crawler.settings.getbool("STATS_DUMP")
self._stats: StatsT = {}
self._crawler: Crawler = crawler
def __getattribute__(self, name):
original_attr = super().__getattribute__(name)
if name in (
"get_value",
"get_stats",
"set_value",
"set_stats",
"inc_value",
"max_value",
"min_value",
"clear_stats",
"open_spider",
"close_spider",
) and callable(original_attr):
def _deprecated_wrapper(*args, **kwargs):
sig = inspect.signature(original_attr).bind(*args, **kwargs)
sig.apply_defaults()
if sig.arguments.get("spider"):
warnings.warn(
f"Passing a 'spider' argument to StatsCollector.{name}() is deprecated and"
f" the argument will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return original_attr(*args, **kwargs)
return _deprecated_wrapper
return original_attr
def get_value(
self, key: str, default: Any = None, spider: Spider | None = None
@ -53,18 +92,20 @@ class StatsCollector:
def clear_stats(self, spider: Spider | None = None) -> None:
self._stats.clear()
def open_spider(self, spider: Spider) -> None:
def open_spider(self, spider: Spider | None = None) -> None:
pass
def close_spider(self, spider: Spider, reason: str) -> None:
def close_spider(
self, spider: Spider | None = None, reason: str | None = None
) -> None:
if self._dump:
logger.info(
"Dumping Scrapy stats:\n" + pprint.pformat(self._stats),
extra={"spider": spider},
extra={"spider": self._crawler.spider},
)
self._persist_stats(self._stats, spider)
self._persist_stats(self._stats)
def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
def _persist_stats(self, stats: StatsT) -> None:
pass
@ -73,8 +114,9 @@ class MemoryStatsCollector(StatsCollector):
super().__init__(crawler)
self.spider_stats: dict[str, StatsT] = {}
def _persist_stats(self, stats: StatsT, spider: Spider) -> None:
self.spider_stats[spider.name] = stats
def _persist_stats(self, stats: StatsT) -> None:
if self._crawler.spider:
self.spider_stats[self._crawler.spider.name] = stats
class DummyStatsCollector(StatsCollector):

4
tests/test_downloadermiddleware_httpcache.py
View File

@ -31,10 +31,10 @@ class TestBase:
body=b"test body",
status=202,
)
self.crawler.stats.open_spider(self.spider)
self.crawler.stats.open_spider()
def teardown_method(self):
self.crawler.stats.close_spider(self.spider, "")
self.crawler.stats.close_spider()
shutil.rmtree(self.tmpdir)
def _get_settings(self, **new_settings):

6
tests/test_downloadermiddleware_httpcompression.py
View File

@ -72,7 +72,7 @@ class TestHttpCompression:
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.crawler.stats.open_spider(self.spider)
self.crawler.stats.open_spider()
def _getresponse(self, coding):
if coding not in FORMAT:
@ -97,8 +97,8 @@ class TestHttpCompression:
return response
def assertStatsEqual(self, key, value):
assert self.crawler.stats.get_value(key, spider=self.spider) == value, str(
self.crawler.stats.get_stats(self.spider)
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
def test_setting_false_compression_enabled(self):

8
tests/test_downloadermiddleware_stats.py
View File

@ -14,14 +14,14 @@ class TestDownloaderStats:
self.spider = self.crawler._create_spider("scrapytest.org")
self.mw = DownloaderStats(self.crawler.stats)
self.crawler.stats.open_spider(self.spider)
self.crawler.stats.open_spider()
self.req = Request("http://scrapytest.org")
self.res = Response("scrapytest.org", status=400)
def assertStatsEqual(self, key, value):
assert self.crawler.stats.get_value(key, spider=self.spider) == value, str(
self.crawler.stats.get_stats(self.spider)
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
def test_process_request(self):
@ -41,4 +41,4 @@ class TestDownloaderStats:
)
def teardown_method(self):
self.crawler.stats.close_spider(self.spider, "")
self.crawler.stats.close_spider()

10
tests/test_spidermiddleware_depth.py
View File

@ -28,13 +28,13 @@ def spider(crawler: Crawler) -> Spider:
@pytest.fixture
def stats(crawler: Crawler, spider: Spider) -> Generator[StatsCollector]:
def stats(crawler: Crawler) -> Generator[StatsCollector]:
assert crawler.stats is not None
crawler.stats.open_spider(spider)
crawler.stats.open_spider()
yield crawler.stats
crawler.stats.close_spider(spider, "")
crawler.stats.close_spider()
@pytest.fixture
@ -53,7 +53,7 @@ def test_process_spider_output(
out = list(mw.process_spider_output(resp, result, spider))
assert out == result
rdc = stats.get_value("request_depth_count/1", spider=spider)
rdc = stats.get_value("request_depth_count/1")
assert rdc == 1
req.meta["depth"] = 1
@ -61,5 +61,5 @@ def test_process_spider_output(
out2 = list(mw.process_spider_output(resp, result, spider))
assert not out2
rdm = stats.get_value("request_depth_max", spider=spider)
rdm = stats.get_value("request_depth_max")
assert rdm == 1

2
tests/test_spidermiddleware_urllength.py
View File

@ -59,6 +59,6 @@ def test_logging(
) -> None:
with caplog.at_level(INFO):
process_spider_output(mw, spider)
ric = stats.get_value("urllength/request_ignored_count", spider=spider)
ric = stats.get_value("urllength/request_ignored_count")
assert ric == 1
assert f"Ignoring link (url length > {maxlength})" in caplog.text

22
tests/test_stats.py
View File

@ -6,6 +6,7 @@ from unittest import mock
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
@ -91,7 +92,7 @@ class TestStatsCollector:
stats.min_value("test4", 7)
assert stats.get_value("test4") == 7
def test_dummy_collector(self, crawler: Crawler, spider: Spider) -> None:
def test_dummy_collector(self, crawler: Crawler) -> None:
stats = DummyStatsCollector(crawler)
assert stats.get_stats() == {}
assert stats.get_value("anything") is None
@ -100,7 +101,20 @@ class TestStatsCollector:
stats.inc_value("v1")
stats.max_value("v2", 100)
stats.min_value("v3", 100)
stats.open_spider(spider)
stats.set_value("test", "value", spider=spider)
stats.open_spider()
stats.set_value("test", "value")
assert stats.get_stats() == {}
assert stats.get_stats(spider) == {}
def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None:
stats = StatsCollector(crawler)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a 'spider' argument to StatsCollector.set_value\(\) is deprecated",
):
stats.set_value("test", "value", spider=spider)
assert stats.get_stats() == {"test": "value"}
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a 'spider' argument to StatsCollector.get_stats\(\) is deprecated",
):
assert stats.get_stats(spider) == {"test": "value"}