From 13be37e4b1076643d2914bb891ba68dd84675d99 Mon Sep 17 00:00:00 2001 From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:07:32 +0300 Subject: [PATCH] Add type hints to test_link.py and test_downloadermiddleware_stats.py (#7785) --- pyproject.toml | 3 --- tests/test_downloadermiddleware_stats.py | 25 +++++++++++++++--------- tests/test_link.py | 16 ++++++++------- tests/test_logstats.py | 11 +++++++---- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4220b12ed..5ed07a9ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,7 +136,6 @@ module = [ "tests.test_downloadermiddleware_redirect_metarefresh", "tests.test_downloadermiddleware_retry", "tests.test_downloadermiddleware_robotstxt", - "tests.test_downloadermiddleware_stats", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -154,11 +153,9 @@ module = [ "tests.test_http_response", "tests.test_http_response_text", "tests.test_item", - "tests.test_link", "tests.test_linkextractors", "tests.test_loader", "tests.test_logformatter", - "tests.test_logstats", "tests.test_mail", "tests.test_pipeline_crawl", "tests.test_pipeline_files", diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 5609360a7..b8fda25b4 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size @@ -12,8 +14,9 @@ class MyException(Exception): class TestDownloaderStats: - def setup_method(self): + def setup_method(self) -> None: self.crawler = get_crawler(Spider) + assert self.crawler.stats is not None self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider() @@ -21,20 +24,21 @@ class TestDownloaderStats: self.req = Request("http://scrapytest.org") self.res = Response("http://scrapytest.org", status=400) - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: object) -> None: + assert self.crawler.stats is not None assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) - def test_process_request(self): + def test_process_request(self) -> None: self.mw.process_request(self.req) self.assertStatsEqual("downloader/request_count", 1) - def test_process_response(self): + def test_process_response(self) -> None: self.mw.process_response(self.req, self.res) self.assertStatsEqual("downloader/response_count", 1) - def test_process_exception(self): + def test_process_exception(self) -> None: self.mw.process_exception(self.req, MyException()) self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual( @@ -42,14 +46,17 @@ class TestDownloaderStats: 1, ) - def test_from_crawler_not_configured(self): + def test_from_crawler_not_configured(self) -> None: crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False}) with pytest.raises(NotConfigured): DownloaderStats.from_crawler(crawler) - def teardown_method(self): + def teardown_method(self) -> None: + assert self.crawler.stats is not None self.crawler.stats.close_spider() -def test_get_header_size_non_list_value(): - assert get_header_size({"Content-Type": "text/html"}) == 0 +def test_get_header_size_non_list_value() -> None: + # Deliberately passing a non-list/tuple header value to make sure + # get_header_size() degrades gracefully instead of raising. + assert get_header_size({"Content-Type": "text/html"}) == 0 # type: ignore[dict-item] diff --git a/tests/test_link.py b/tests/test_link.py index 0eeffe12b..40c53edd3 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,18 +1,20 @@ +from __future__ import annotations + import pytest from scrapy.link import Link class TestLink: - def _assert_same_links(self, link1, link2): + def _assert_same_links(self, link1: Link, link2: Link) -> None: assert link1 == link2 assert hash(link1) == hash(link2) - def _assert_different_links(self, link1, link2): + def _assert_different_links(self, link1: Link, link2: Link) -> None: assert link1 != link2 assert hash(link1) != hash(link2) - def test_eq_and_hash(self): + def test_eq_and_hash(self) -> None: l1 = Link("http://www.example.com") l2 = Link("http://www.example.com/other") l3 = Link("http://www.example.com") @@ -45,17 +47,17 @@ class TestLink: self._assert_different_links(l7, l9) self._assert_different_links(l7, l10) - def test_repr(self): + def test_repr(self) -> None: l1 = Link( "http://www.example.com", text="test", fragment="something", nofollow=True ) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_bytes_url(self): + def test_bytes_url(self) -> None: with pytest.raises(TypeError): - Link(b"http://www.example.com/\xc2\xa3") + Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type] - def test_eq_non_link(self): + def test_eq_non_link(self) -> None: url = "http://example.com" assert Link(url) != url diff --git a/tests/test_logstats.py b/tests/test_logstats.py index 370728e6a..213681ad1 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime import pytest @@ -9,16 +11,17 @@ from tests.utils.decorators import coroutine_test class TestLogStats: - def setup_method(self): + def setup_method(self) -> None: self.crawler = get_crawler(SimpleSpider) self.spider = self.crawler._create_spider("spidey") + assert self.crawler.stats is not None self.stats = self.crawler.stats self.stats.set_value("response_received_count", 4802) self.stats.set_value("item_scraped_count", 3201) @coroutine_test - async def test_stats_calculations(self): + async def test_stats_calculations(self) -> None: logstats = LogStats.from_crawler(self.crawler) with pytest.raises(AttributeError): @@ -56,7 +59,7 @@ class TestLogStats: assert self.stats.get_value("responses_per_minute") == 172.9 assert self.stats.get_value("items_per_minute") == 116.4 - def test_stats_calculations_no_time(self): + def test_stats_calculations_no_time(self) -> None: """The stat values should be None since the start and finish time are not available. """ @@ -65,7 +68,7 @@ class TestLogStats: assert self.stats.get_value("responses_per_minute") is None assert self.stats.get_value("items_per_minute") is None - def test_stats_calculation_no_elapsed_time(self): + def test_stats_calculation_no_elapsed_time(self) -> None: """The stat values should be None since the elapsed time is 0.""" logstats = LogStats.from_crawler(self.crawler) self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))