mirror of https://github.com/scrapy/scrapy.git
Use autodoc for stats.rst (#7769)
This commit is contained in:
parent
67e5282684
commit
abbc024bbe
|
|
@ -94,27 +94,10 @@ default Stats Collector used is the :class:`MemoryStatsCollector`.
|
||||||
MemoryStatsCollector
|
MemoryStatsCollector
|
||||||
--------------------
|
--------------------
|
||||||
|
|
||||||
.. class:: MemoryStatsCollector
|
.. autoclass:: MemoryStatsCollector
|
||||||
|
:members:
|
||||||
A simple stats collector that keeps the stats of the last scraping run (for
|
|
||||||
each spider) in memory, after they're closed. The stats can be accessed
|
|
||||||
through the :attr:`spider_stats` attribute, which is a dict keyed by spider
|
|
||||||
name.
|
|
||||||
|
|
||||||
This is the default Stats Collector used in Scrapy.
|
|
||||||
|
|
||||||
.. attribute:: spider_stats
|
|
||||||
|
|
||||||
A dict of dicts (keyed by spider name) containing the stats of the last
|
|
||||||
scraping run for each spider.
|
|
||||||
|
|
||||||
DummyStatsCollector
|
DummyStatsCollector
|
||||||
-------------------
|
-------------------
|
||||||
|
|
||||||
.. class:: DummyStatsCollector
|
.. autoclass:: DummyStatsCollector
|
||||||
|
|
||||||
A Stats collector which does nothing but is very efficient (because it does
|
|
||||||
nothing). This stats collector can be set via the :setting:`STATS_CLASS`
|
|
||||||
setting, to disable stats collect in order to improve performance. However,
|
|
||||||
the performance penalty of stats collection is usually marginal compared to
|
|
||||||
other Scrapy workload like parsing pages.
|
|
||||||
|
|
|
||||||
|
|
@ -101,9 +101,21 @@ class StatsCollector:
|
||||||
|
|
||||||
|
|
||||||
class MemoryStatsCollector(StatsCollector):
|
class MemoryStatsCollector(StatsCollector):
|
||||||
|
"""A simple stats collector that keeps the stats of the last scraping run
|
||||||
|
(for each spider) in memory, after they're closed. The stats can be
|
||||||
|
accessed through the :attr:`spider_stats` attribute, which is a dict keyed
|
||||||
|
by spider name.
|
||||||
|
|
||||||
|
This is the default stats collector used in Scrapy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
spider_stats: dict[str, StatsT]
|
||||||
|
"""A dict of dicts (keyed by spider name) containing the stats of the last
|
||||||
|
scraping run for each spider."""
|
||||||
|
|
||||||
def __init__(self, crawler: Crawler):
|
def __init__(self, crawler: Crawler):
|
||||||
super().__init__(crawler)
|
super().__init__(crawler)
|
||||||
self.spider_stats: dict[str, StatsT] = {}
|
self.spider_stats = {}
|
||||||
|
|
||||||
def _persist_stats(self, stats: StatsT) -> None:
|
def _persist_stats(self, stats: StatsT) -> None:
|
||||||
if self._crawler.spider:
|
if self._crawler.spider:
|
||||||
|
|
@ -111,6 +123,13 @@ class MemoryStatsCollector(StatsCollector):
|
||||||
|
|
||||||
|
|
||||||
class DummyStatsCollector(StatsCollector):
|
class DummyStatsCollector(StatsCollector):
|
||||||
|
"""A stats collector which does nothing but is very efficient (because it
|
||||||
|
does nothing). This stats collector can be set via the
|
||||||
|
:setting:`STATS_CLASS` setting, to disable stats collection in order to
|
||||||
|
improve performance. However, the performance penalty of stats collection
|
||||||
|
is usually marginal compared to other Scrapy workload like parsing pages.
|
||||||
|
"""
|
||||||
|
|
||||||
def get_value(
|
def get_value(
|
||||||
self, key: str, default: Any = None, spider: Spider | None = None
|
self, key: str, default: Any = None, spider: Spider | None = None
|
||||||
) -> Any:
|
) -> Any:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue