RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

killall unnecessary-asserts

This commit is contained in:
Adrian Chaves 2026-08-10 17:41:08 +02:00
parent 9f416fbb49
commit dcf2483f8d
31 changed files with 0 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/benchmarks/__init__.py
View File

@ -36,7 +36,6 @@ class NullDownloadHandler:
async def download_request(self, request: Request) -> Response:
self._active += 1
assert self._crawler.stats
self._crawler.stats.max_value("benchmark/peak_concurrency", self._active)
try:
await asyncio.sleep(0)

5
tests/benchmarks/test_crawl.py
View File

@ -128,7 +128,6 @@ class _DelayedPipeline:
async def process_item(self, item: Any) -> Any:
url = item["url"]
self._active[url] += 1
assert self._crawler.stats
self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
try:
await asyncio.sleep(DELAY)
@ -147,7 +146,6 @@ def _crawl_tree(
pages=pages,
items=items,
)
assert crawler.stats
assert crawler.stats.get_value("downloader/response_count") == domains * pages
assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
return crawler
@ -166,7 +164,6 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> N
def run() -> None:
crawler = crawl(_FollowSpider, settings, url=url)
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
benchmark(run)
@ -177,7 +174,6 @@ def test_overhead_engine(benchmark: BenchmarkFixture) -> None:
def run() -> None:
crawler = _crawl_tree({}, domains=1, pages=REQUESTS)
assert crawler.stats
assert crawler.stats.get_value("benchmark/peak_concurrency") > 1
benchmark(run)
@ -262,7 +258,6 @@ def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
crawler = _crawl_tree(
settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
)
assert crawler.stats
assert (
crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
)

7
tests/test_closespider.py
View File

@ -28,7 +28,6 @@ class TestCloseSpider:
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, ItemSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_itemcount"
itemcount = crawler.stats.get_value("item_scraped_count")
@ -40,7 +39,6 @@ class TestCloseSpider:
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount"
pagecount = crawler.stats.get_value("response_received_count")
@ -61,7 +59,6 @@ class TestCloseSpider:
max_items=max_items, max_requests=max_requests, mockserver=self.mockserver
)
assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@ -81,7 +78,6 @@ class TestCloseSpider:
)
yield crawler.crawl(mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@ -93,7 +89,6 @@ class TestCloseSpider:
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
assert isinstance(crawler.spider, ErrorSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_errorcount"
key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
@ -107,7 +102,6 @@ class TestCloseSpider:
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
assert isinstance(crawler.spider, FollowAllSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
@ -119,7 +113,6 @@ class TestCloseSpider:
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
yield crawler.crawl(n=3, mockserver=self.mockserver)
assert isinstance(crawler.spider, SlowSpider)
assert crawler.stats
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout_no_item"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")

4
tests/test_crawl.py
View File

@ -359,7 +359,6 @@ with multiples lines
est: list[list[tuple[str, Any]]] = []
def cb(response):
assert crawler.engine
est.append(get_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@ -377,7 +376,6 @@ with multiples lines
est: list[str] = []
def cb(response):
assert crawler.engine
est.append(format_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@ -452,7 +450,6 @@ class TestCrawlSpider:
await crawler.crawl_async(
mockserver.url("/status?n=200"), mockserver=mockserver
)
assert crawler.stats
return items, crawler.stats
@coroutine_test
@ -518,7 +515,6 @@ class TestCrawlSpider:
# so the failure is dropped silently and the crawl finishes normally.
assert "[parse] status 200 (foo: None)" in caplog.text
assert "[errback]" not in caplog.text
assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/404") == 1
@coroutine_test

3
tests/test_crawler.py
View File

@ -536,7 +536,6 @@ class TestSpiderSettings:
custom_settings = {"AUTOTHROTTLE_ENABLED": True}
crawler = get_crawler(MySpider)
assert crawler.extensions
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
assert AutoThrottle in enabled_exts
@ -568,7 +567,6 @@ class TestCrawlerLogging:
}
async def start(self):
assert crawler.stats
info_count_start = crawler.stats.get_value("log_count/INFO")
logging.debug("debug message") # noqa: LOG015
logging.info("info message") # noqa: LOG015
@ -601,7 +599,6 @@ class TestCrawlerLogging:
assert "info message" in logged
assert "warning message" in logged
assert "error message" in logged
assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
assert crawler.stats.get_value("log_count/WARNING") == 1
assert info_count == 1

3
tests/test_downloadermiddleware_httpcache.py
View File

@ -71,7 +71,6 @@ class TestBase:
settings = self._get_settings(**new_settings)
crawler = get_crawler(Spider, settings)
crawler.spider = crawler._create_spider("example.com")
assert crawler.stats
crawler.stats.open_spider()
try:
yield crawler
@ -136,7 +135,6 @@ class StorageTestMixin(TestBase):
with self._middleware() as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -158,7 +156,6 @@ class StorageTestMixin(TestBase):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)

2
tests/test_downloadermiddleware_httpcompression.py
View File

@ -60,7 +60,6 @@ class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
def _getresponse(self, coding: str) -> Response:
@ -86,7 +85,6 @@ class TestHttpCompression:
return response
def assertStatsEqual(self, key: str, value: Any) -> None:
assert self.crawler.stats
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)

1
tests/test_downloadermiddleware_offsite.py
View File

@ -229,7 +229,6 @@ def test_repeated_offsite_domain():
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
assert crawler.stats
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):

12
tests/test_downloadermiddleware_retry.py
View File

@ -81,7 +81,6 @@ class TestRetry:
# discard it
assert self.mw.process_response(req3, rsp) is rsp
assert self.crawler.stats
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
@ -133,7 +132,6 @@ class TestRetry:
self._test_retry_exception(req, exc("foo"))
stats = self.crawler.stats
assert stats
assert stats.get_value("retry/max_reached") == len(exceptions)
assert stats.get_value("retry/count") == len(exceptions) * 2
assert (
@ -314,7 +312,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -1
expected_reason = "unspecified"
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@ -335,7 +332,6 @@ class TestGetRetryRequest:
max_retry_times=max_retry_times,
)
assert new_request is None
assert spider.crawler.stats
assert spider.crawler.stats.get_value("retry/max_reached") == 1
failure_count = max_retry_times + 1
expected_reason = "unspecified"
@ -362,7 +358,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -1
expected_reason = "unspecified"
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@ -395,7 +390,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -expected_retry_times
expected_reason = "unspecified"
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
value = spider.crawler.stats.get_value(stat)
assert value == expected_retry_times
@ -520,7 +514,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@ -542,7 +535,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -568,7 +560,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -592,7 +583,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -618,7 +608,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@ -736,7 +725,6 @@ class TestGetRetryRequest:
reason=expected_reason,
stats_base_key=stats_key,
)
assert spider.crawler.stats
for stat in (
f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}",

3
tests/test_downloadermiddleware_stats.py
View File

@ -16,7 +16,6 @@ class MyException(Exception):
class TestDownloaderStats:
def setup_method(self) -> None:
self.crawler = get_crawler(Spider)
assert self.crawler.stats is not None
self.mw = DownloaderStats(self.crawler.stats)
self.crawler.stats.open_spider()
@ -25,7 +24,6 @@ class TestDownloaderStats:
self.res = Response("http://scrapytest.org", status=400)
def assertStatsEqual(self, key: str, value: object) -> None:
assert self.crawler.stats is not None
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
@ -52,7 +50,6 @@ class TestDownloaderStats:
DownloaderStats.from_crawler(crawler)
def teardown_method(self) -> None:
assert self.crawler.stats is not None
self.crawler.stats.close_spider()

1
tests/test_downloaderslotssettings.py
View File

@ -149,7 +149,6 @@ async def test_none_slot_with_priority_queue(
assert crawler.spider.default_slot in crawler.spider.times
assert len(crawler.spider.times[crawler.spider.default_slot]) == 2
assert crawler.stats
stats = crawler.stats
assert stats.get_value("spider_exceptions", 0) == 0
assert stats.get_value("downloader/exception_count", 0) == 0

2
tests/test_dupefilters.py
View File

@ -178,7 +178,6 @@ class TestRFPDupeFilter:
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats
assert crawler.stats.get_value("dupefilter/filtered") == 2
assert (
"scrapy.dupefilters",
@ -212,7 +211,6 @@ class TestRFPDupeFilter:
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats
assert crawler.stats.get_value("dupefilter/filtered") == 2
assert (
"scrapy.dupefilters",

11
tests/test_engine_loop.py
View File

@ -65,7 +65,6 @@ class TestMain:
name = "test"
async def start(self) -> AsyncIterator[Any]:
assert self.crawler.engine
assert self.crawler.engine._slot
scheduler = self.crawler.engine._slot.scheduler
assert isinstance(scheduler, MemoryScheduler)
@ -111,7 +110,6 @@ class TestMain:
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler.signals.connect(track_url, signals.request_reached_downloader)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"]
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
@ -124,7 +122,6 @@ class TestMain:
name = "test"
async def start(self) -> AsyncIterator[Any]:
assert self.crawler.engine is not None
await self.crawler.engine.close_async()
yield Request("data:,a")
@ -145,7 +142,6 @@ class TestMain:
await crawler.crawl_async()
assert not caplog.records
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "shutdown"
assert not actual_urls
@ -186,7 +182,6 @@ class TestMain:
failure, response = errors[0]
assert isinstance(failure.value, ValueError)
assert response is None
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "start_error"
assert crawler.stats.get_value("spider_exceptions/count") == 1
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
@ -213,7 +208,6 @@ class TestMain:
await crawler.crawl_async()
assert not caplog.records
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "my_reason"
assert crawler.stats.get_value("spider_exceptions/count") is None
@ -296,7 +290,6 @@ class TestRequestSendOrder:
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler.signals.connect(track_num, signals.request_reached_downloader)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
expected_nums = sorted(start_nums + cb_nums)
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"
@ -316,7 +309,6 @@ class TestRequestSendOrder:
)
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
@ -358,7 +350,6 @@ class TestRequestSendOrder:
)
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
@ -403,7 +394,6 @@ class TestRequestSendOrder:
)
async def start(spider: Spider) -> AsyncIterator[Any]:
assert spider.crawler.engine
assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
@ -460,7 +450,6 @@ class TestRequestSendOrder:
async def start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
assert spider.crawler.engine
if spider.crawler.engine.needs_backout():
await spider.crawler.signals.wait_for(signals.scheduler_empty)
request = self.request(num, response_seconds, download_slots)

2
tests/test_extension_memdebug.py
View File

@ -32,7 +32,6 @@ def test_spider_closed_sets_stats() -> None:
ext.spider_closed(DefaultSpider(), "finished")
assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)
assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked)
assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None
@ -46,6 +45,5 @@ async def test_crawl_sets_stats() -> None:
crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True})
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None
assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1

3
tests/test_extension_memusage.py
View File

@ -98,7 +98,6 @@ async def test_memusage_below_thresholds_logs_peak(
with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=1)
assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") is None
assert crawler.stats.get_value("memusage/warning_reached") is None
assert crawler.stats.get_value("memusage/max") == 25 * MB
@ -128,7 +127,6 @@ async def test_memusage_limit_closes_spider_with_reason_and_error_log(
with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=100)
assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") == 1
assert crawler.stats.get_value("finish_reason") == "memusage_exceeded"
assert any(
@ -166,7 +164,6 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
await crawler.crawl_async(url="data:,", loops=60)
assert warning_signals == [1]
assert crawler.stats
assert crawler.stats.get_value("memusage/warning_reached") == 1
assert crawler.stats.get_value("finish_reason") == "finished"
warnings_logged = [

1
tests/test_extension_telnet.py
View File

@ -118,7 +118,6 @@ async def test_telnet_vars() -> None:
start_urls = ["data:,"]
async def parse(self, response: Response) -> None:
assert self.crawler.extensions
console = next(
ext
for ext in self.crawler.extensions.middlewares

3
tests/test_feedexport.py
View File

@ -309,7 +309,6 @@ class TestFeedExport(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.stats is not None
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
@ -333,7 +332,6 @@ class TestFeedExport(TestFeedExportBase):
side_effect=store,
):
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.stats is not None
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
@ -351,7 +349,6 @@ class TestFeedExport(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.stats is not None
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1

2
tests/test_feedexport_batch.py
View File

@ -419,7 +419,6 @@ class TestBatchDeliveries(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(total=2, mockserver=self.mockserver)
assert crawler.stats
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@ -490,7 +489,6 @@ class TestBatchDeliveries(TestFeedExportBase):
assert len(CustomS3FeedStorage.stubs) == len(items)
for stub in CustomS3FeedStorage.stubs:
stub.assert_no_pending_responses()
assert crawler.stats
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
)

1
tests/test_logstats.py
View File

@ -14,7 +14,6 @@ class TestLogStats:
def setup_method(self) -> None:
self.crawler = get_crawler(SimpleSpider)
self.spider = self.crawler._create_spider("spidey")
assert self.crawler.stats is not None
self.stats = self.crawler.stats
self.stats.set_value("response_received_count", 4802)

2
tests/test_pipeline_crawl.py
View File

@ -140,7 +140,6 @@ class TestFileDownloadCrawl:
assert not items[0][self.media_key]
# check that there was 1 successful fetch and 3 other responses with non-200 code
assert crawler.stats
assert crawler.stats.get_value("downloader/request_method_count/GET") == 4
assert crawler.stats.get_value("downloader/response_count") == 4
assert crawler.stats.get_value("downloader/response_status_count/200") == 1
@ -208,7 +207,6 @@ class TestFileDownloadCrawl:
mockserver=self.mockserver,
)
self._assert_files_downloaded(self.items, caplog.text)
assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/302") == 3
@coroutine_test

1
tests/test_pipeline_media.py
View File

@ -81,7 +81,6 @@ class TestBaseMediaPipeline:
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
assert crawler.request_fingerprinter is not None
self.fingerprint = crawler.request_fingerprinter.fingerprint
@property

2
tests/test_request_cb_kwargs.py
View File

@ -70,7 +70,6 @@ class KeywordArgumentsSpider(MockServerSpider):
checks: list[bool] = []
def _inc_checks(self, count: int = 1) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("boolean_checks", count)
async def start(self):
@ -171,7 +170,6 @@ class TestCallbackKeywordArguments:
await crawler.crawl_async(mockserver=mockserver)
assert isinstance(crawler.spider, KeywordArgumentsSpider)
assert all(crawler.spider.checks)
assert crawler.stats
assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
# check exceptions for argument mismatch
exceptions = {}

3
tests/test_scheduler.py
View File

@ -54,7 +54,6 @@ async def create_scheduler(
finally:
await ensure_awaitable(scheduler.close("finished"))
await mock_crawler.stop_async()
assert mock_crawler.engine
mock_crawler.engine.downloader.close()
@ -254,7 +253,6 @@ class DownloaderAwareSchedulerTestMixin(TestSchedulerBase):
dequeued_slots: list[str] = []
requests: list[Request] = []
assert scheduler.crawler
assert scheduler.crawler.engine
downloader = scheduler.crawler.engine.downloader
assert isinstance(downloader, MockDownloader)
while scheduler.has_pending_requests():
@ -320,7 +318,6 @@ class TestIntegrationWithDownloaderAwareInMemory:
url = mockserver.url("/status?n=200", is_secure=False)
start_urls = [url] * 6
yield self.crawler.crawl(start_urls)
assert self.crawler.stats
assert self.crawler.stats.get_value("downloader/response_count") == len(
start_urls
)

4
tests/test_spider.py
View File

@ -100,7 +100,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
@coroutine_test
@ -135,7 +134,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1
@pytest.mark.parametrize("feed_iterator", ["xml", "html"])
@ -155,7 +153,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
@ -206,7 +203,6 @@ class TestCSVFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1

1
tests/test_spider_start.py
View File

@ -34,7 +34,6 @@ class TestMain:
crawler = get_crawler(spider)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items

2
tests/test_spidermiddleware_depth.py
View File

@ -26,7 +26,6 @@ def crawler() -> Crawler:
@pytest.fixture
def stats(crawler: Crawler) -> Generator[StatsCollector]:
assert crawler.stats is not None
crawler.stats.open_spider()
yield crawler.stats
@ -120,7 +119,6 @@ def test_priority_and_non_verbose_stats() -> None:
Spider,
{"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},
)
assert crawler.stats is not None
crawler.stats.open_spider()
try:
mw = build_from_crawler(DepthMiddleware, crawler)

1
tests/test_spidermiddleware_httperror.py
View File

@ -204,7 +204,6 @@ class TestHttpErrorMiddlewareIntegrational:
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
assert crawler.stats
get_value = crawler.stats.get_value
assert get_value("httperror/response_ignored_count") == 3
assert get_value("httperror/response_ignored_status_count/404") == 1

1
tests/test_spidermiddleware_process_start.py
View File

@ -80,7 +80,6 @@ class TestMain:
crawler = get_crawler(spider_cls, settings_dict=settings)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"

1
tests/test_spidermiddleware_urllength.py
View File

@ -31,7 +31,6 @@ def crawler() -> Crawler:
@pytest.fixture
def stats(crawler: Crawler) -> StatsCollector:
assert crawler.stats is not None
return crawler.stats

3
tests/test_utils_log.py
View File

@ -89,7 +89,6 @@ class TestLogCounterHandler:
logger.removeHandler(handler)
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None
assert crawler.stats.get_value("log_count/INFO") is None
assert crawler.stats.get_value("log_count/WARNING") is None
@ -98,12 +97,10 @@ class TestLogCounterHandler:
def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.error("test log msg")
assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.debug("test log msg")
assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None

5
tests/test_utils_request.py
View File

@ -274,7 +274,6 @@ class TestRequestFingerprinter:
def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
assert crawler.request_fingerprinter
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
@ -291,7 +290,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@ -314,7 +312,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com?a=1&a=2")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com?a=2&a=1")
@ -333,7 +330,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@ -365,7 +361,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
assert crawler.request_fingerprinter
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]