From 61f99f2df1d373d3387c5d2dffa33b38e14164fb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 8 Jul 2026 13:05:45 +0500 Subject: [PATCH] More granular untyped defs checking in tests. (#7712) --- pyproject.toml | 77 ++++++++++++++++++- .../asyncio_deferred_signal.py | 2 +- .../caching_hostname_resolver.py | 1 + .../CrawlerProcess/asyncio_deferred_signal.py | 2 +- .../caching_hostname_resolver.py | 1 + tests/CrawlerProcess/sleeping.py | 2 +- tests/CrawlerRunner/ip_address.py | 1 + tests/test_addons.py | 6 +- .../__init__.py | 31 ++++---- tests/test_commands.py | 1 + tests/test_crawl.py | 10 ++- tests/test_crawler.py | 1 + tests/test_dependencies.py | 3 + tests/test_downloader_handlers.py | 6 +- tests/test_downloadermiddleware.py | 4 +- ...est_downloadermiddleware_defaultheaders.py | 47 +++++------ ...st_downloadermiddleware_downloadtimeout.py | 72 +++++++++-------- tests/test_downloadermiddleware_useragent.py | 52 +++++++------ tests/test_engine.py | 10 ++- tests/test_extension_periodic_log.py | 8 +- tests/test_feedexport_batch.py | 6 +- tests/test_http_cookies.py | 4 +- tests/test_responsetypes.py | 12 +-- tests/test_scheduler.py | 3 +- tests/test_signals.py | 26 +++---- tests/test_spider_start.py | 11 ++- tests/test_spidermiddleware.py | 2 +- tests/test_spidermiddleware_httperror.py | 3 +- tests/test_spidermiddleware_referer.py | 9 ++- tests/test_spidermiddleware_start.py | 9 ++- tests/test_utils_asyncio.py | 4 +- tests/test_utils_conf.py | 6 +- tests/test_utils_console.py | 3 + tests/test_utils_display.py | 20 ++++- tests/test_utils_iterators.py | 4 +- tests/test_utils_log.py | 2 +- tests/test_utils_misc/__init__.py | 6 +- tests/test_utils_response.py | 4 +- tests/test_utils_signal.py | 12 ++- tests/test_utils_spider.py | 4 +- tests/test_utils_trackref.py | 6 +- 41 files changed, 331 insertions(+), 162 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed2e99bea..ac9d37e78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,82 @@ untyped_calls_exclude = [ [[tool.mypy.overrides]] module = "tests.*" allow_untyped_defs = true -allow_incomplete_defs = true # 48 errors +allow_incomplete_defs = true # 59 errors + +# TODO +[[tool.mypy.overrides]] +module = [ + "tests.mockserver.*", + "tests.spiders", + "tests.test_closespider", + "tests.test_cmdline", + "tests.test_contracts", + "tests.test_core_downloader", + "tests.test_downloader_handler_twisted_ftp", + "tests.test_downloadermiddleware_cookies", + "tests.test_downloadermiddleware_httpauth", + "tests.test_downloadermiddleware_httpcache", + "tests.test_downloadermiddleware_httpcompression", + "tests.test_downloadermiddleware_httpproxy", + "tests.test_downloadermiddleware_offsite", + "tests.test_downloadermiddleware_redirect", + "tests.test_downloadermiddleware_redirect_base", + "tests.test_downloadermiddleware_redirect_metarefresh", + "tests.test_downloadermiddleware_retry", + "tests.test_downloadermiddleware_robotstxt", + "tests.test_downloadermiddleware_stats", + "tests.test_downloaderslotssettings", + "tests.test_dupefilters", + "tests.test_engine_loop", + "tests.test_exporters", + "tests.test_extension_statsmailer", + "tests.test_extension_throttle", + "tests.test_feedexport", + "tests.test_feedexport_postprocess", + "tests.test_feedexport_storages", + "tests.test_feedexport_uri_params", + "tests.test_http2_client_protocol", + "tests.test_http_headers", + "tests.test_http_request", + "tests.test_http_request_form", + "tests.test_http_response", + "tests.test_http_response_text", + "tests.test_item", + "tests.test_link", + "tests.test_linkextractors", + "tests.test_loader", + "tests.test_logformatter", + "tests.test_logstats", + "tests.test_mail", + "tests.test_pipeline_crawl", + "tests.test_pipeline_files", + "tests.test_pipeline_images", + "tests.test_pipeline_media", + "tests.test_pipelines", + "tests.test_pqueues", + "tests.test_request_attribute_binding", + "tests.test_request_cb_kwargs", + "tests.test_request_dict", + "tests.test_request_left", + "tests.test_robotstxt_interface", + "tests.test_scheduler_base", + "tests.test_settings", + "tests.test_spider", + "tests.test_spider_crawl", + "tests.test_spidermiddleware_output_chain", + "tests.test_spidermiddleware_process_start", + "tests.test_spider_sitemap", + "tests.test_squeues", + "tests.test_squeues_request", + "tests.test_stats", + "tests.test_utils_datatypes", + "tests.test_utils_decorators", + "tests.test_utils_defer", + "tests.test_utils_deprecate", + "tests.test_utils_misc.test_return_with_argument_inside_generator", + "tests.test_utils_python", + "tests.test_utils_request", +] check_untyped_defs = false # Interface classes are hard to support diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py index e000a23db..2f4f2d051 100644 --- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py +++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py @@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro class UppercasePipeline: - async def _open_spider(self, spider): + async def _open_spider(self, spider: Spider) -> None: spider.logger.info("async pipeline opened!") await asyncio.sleep(0.1) diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py index a6520ca22..7e3988d92 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ name = "caching_hostname_resolver_spider" + url: str async def start(self): yield scrapy.Request(self.url) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index eb2e1105c..6e4db1990 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro class UppercasePipeline: - async def _open_spider(self, spider): + async def _open_spider(self, spider: Spider) -> None: spider.logger.info("async pipeline opened!") await asyncio.sleep(0.1) diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 7f687e7d9..79ce9c707 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ name = "caching_hostname_resolver_spider" + url: str async def start(self): yield scrapy.Request(self.url) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index a577b1909..9f7d14606 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -15,7 +15,7 @@ class SleepingSpider(scrapy.Spider): async def parse(self, response): from twisted.internet import reactor - d = Deferred() + d: Deferred[None] = Deferred() reactor.callLater(int(sys.argv[1]), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index fae206610..bab4f65db 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -36,6 +36,7 @@ def createResolver(servers: list[tuple[str, int]]) -> ResolverBase: class LocalhostSpider(Spider): name = "localhost_spider" + url: str async def start(self): yield Request(self.url) diff --git a/tests/test_addons.py b/tests/test_addons.py index 14ebddda8..a041af6d6 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -107,11 +107,11 @@ class TestAddonManager: addonlist = [] for i in range(3): addon = get_addon_cls({"KEY1": i}) - addon.number = i + addon.number = i # type: ignore[attr-defined] addonlist.append(addon) # Test for every possible ordering for ordered_addons in itertools.permutations(addonlist): - expected_order = [a.number for a in ordered_addons] + expected_order = [a.number for a in ordered_addons] # type: ignore[attr-defined] settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} crawler = get_crawler(settings_dict=settings) manager = crawler.addons @@ -177,7 +177,7 @@ class TestAddonManager: ) settings["SCHEDULER"] = "AddonScheduler" - settings_dict = { + settings_dict: dict[str, Any] = { "ADDONS": {AddonWithFallback: 1}, } crawler = get_crawler(settings_dict=settings_dict) diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index f17543575..50b04de0b 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,21 +1,24 @@ +from __future__ import annotations + import sys from pathlib import Path from subprocess import PIPE, Popen -class TestCmdlineCrawlPipeline: - def _execute(self, spname): - args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) - cwd = Path(__file__).resolve().parent - proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) - _, stderr = proc.communicate() - return proc.returncode, stderr +def _execute(spname: str) -> int: + args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) + cwd = Path(__file__).resolve().parent + proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) + proc.communicate() + return proc.returncode - def test_open_spider_normally_in_pipeline(self): - returncode, _ = self._execute("normal") - assert returncode == 0 - def test_exception_at_open_spider_in_pipeline(self): - returncode, _ = self._execute("exception") - # An exception in pipeline's open_spider should result in a non-zero exit code - assert returncode == 1 +def test_open_spider_normally_in_pipeline(): + returncode = _execute("normal") + assert returncode == 0 + + +def test_exception_at_open_spider_in_pipeline(): + returncode = _execute("exception") + # An exception in pipeline's open_spider should result in a non-zero exit code + assert returncode == 1 diff --git a/tests/test_commands.py b/tests/test_commands.py index 9f81e8602..bb4d96f00 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -89,6 +89,7 @@ class TestCommandSettings: args=["-s", f"FEEDS={feeds_json}", "spider.py"] ) self.command.process_options(args, opts) + assert self.command.settings is not None assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings) assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index be6c80429..8c1716345 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -356,9 +356,10 @@ with multiples lines @coroutine_test async def test_engine_status(self, mockserver: MockServer) -> None: - est = [] + est: list[list[tuple[str, Any]]] = [] def cb(response): + assert crawler.engine est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) @@ -373,9 +374,10 @@ with multiples lines @coroutine_test async def test_format_engine_status(self, mockserver: MockServer) -> None: - est = [] + est: list[str] = [] def cb(response): + assert crawler.engine est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) @@ -386,8 +388,8 @@ with multiples lines assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict - est = [x.split(":") for x in est] - est = [x for sublist in est for x in sublist] # flatten + est_split = [x.split(":") for x in est] + est = [x for sublist in est_split for x in sublist] # flatten est = [x.lstrip().rstrip() for x in est] it = iter(est) s = dict(zip(it, it, strict=True)) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index adac32df1..6a1eea027 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -523,6 +523,7 @@ class TestCrawlerLogging: } async def start(self): + assert crawler.stats info_count_start = crawler.stats.get_value("log_count/INFO") logging.debug("debug message") # noqa: LOG015 logging.info("info message") # noqa: LOG015 diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 578e84359..41c7ce526 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re from configparser import ConfigParser @@ -23,6 +25,7 @@ class TestScrapyUtils: config_parser.read(tox_config_file_path) pattern = r"Twisted==([\d.]+)" match = re.search(pattern, config_parser["min"]["deps"]) + assert match pinned_twisted_version_string = match[1] assert twisted_version.short() == pinned_twisted_version_string diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 275199a76..4af5ca0e8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -16,7 +16,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request +from scrapy.http import Request, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.boto import is_botocore_available from scrapy.utils.misc import build_from_crawler @@ -354,6 +354,7 @@ class TestDataURI: response = await self.download_request(request) assert response.text == "A brief note" assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert isinstance(response, TextResponse) assert response.encoding == "US-ASCII" @coroutine_test @@ -362,6 +363,7 @@ class TestDataURI: response = await self.download_request(request) assert response.text == "\u038e\u03a3\u038e" assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert isinstance(response, TextResponse) assert response.encoding == "iso-8859-7" @coroutine_test @@ -370,6 +372,7 @@ class TestDataURI: response = await self.download_request(request) assert response.text == "\u038e\u03a3\u038e" assert response.body == b"\xbe\xd3\xbe" + assert isinstance(response, TextResponse) assert response.encoding == "iso-8859-7" @coroutine_test @@ -382,6 +385,7 @@ class TestDataURI: response = await self.download_request(request) assert response.text == "\u038e\u03a3\u038e" assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert isinstance(response, TextResponse) assert response.encoding == "utf-8" @coroutine_test diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index d321c9047..92eb18d33 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -233,7 +233,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): return result def process_request(self, request): - d = Deferred() + d: Deferred[Response] = Deferred() d.addCallback(self.cb) d.callback(resp) return d @@ -298,6 +298,8 @@ class TestDownloadDeprecated(TestManagerBase): return succeed(resp) async with self.get_mwman() as mwman: + assert mwman.crawler + assert mwman.crawler.spider with pytest.warns( ScrapyDeprecationWarning, match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead", diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index e697de9b7..8c89c3ffb 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider @@ -5,28 +7,29 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -class TestDefaultHeadersMiddleware: - def get_defaults_mw(self): - crawler = get_crawler(Spider) - defaults = { - to_bytes(k): [to_bytes(v)] - for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() - } - return defaults, DefaultHeadersMiddleware.from_crawler(crawler) +def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddleware]: + crawler = get_crawler(Spider) + defaults = { + to_bytes(k): [to_bytes(v)] + for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() + } + return defaults, DefaultHeadersMiddleware.from_crawler(crawler) - def test_process_request(self): - defaults, mw = self.get_defaults_mw() - req = Request("http://www.scrapytest.org") - mw.process_request(req) - assert req.headers == defaults - def test_update_headers(self): - defaults, mw = self.get_defaults_mw() - headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} - bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} - req = Request("http://www.scrapytest.org", headers=headers) - assert req.headers == bytes_headers +def test_process_request(): + defaults, mw = get_defaults_mw() + req = Request("http://www.scrapytest.org") + mw.process_request(req) + assert req.headers == defaults - mw.process_request(req) - defaults.update(bytes_headers) - assert req.headers == defaults + +def test_update_headers(): + defaults, mw = get_defaults_mw() + headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} + bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} + req = Request("http://www.scrapytest.org", headers=headers) + assert req.headers == bytes_headers + + mw.process_request(req) + defaults.update(bytes_headers) + assert req.headers == defaults diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index e6b17960e..9b64cf349 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,43 +1,51 @@ +from __future__ import annotations + +from typing import Any + from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestDownloadTimeoutMiddleware: - def get_request_spider_mw(self, settings=None): - crawler = get_crawler(Spider, settings) - spider = crawler._create_spider("foo") - request = Request("http://scrapytest.org/") - return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) +def get_request_spider_mw(settings: dict[str, Any] | None = None): + crawler = get_crawler(Spider, settings) + spider = crawler._create_spider("foo") + request = Request("http://scrapytest.org/") + return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) - def test_default_download_timeout(self): - req, spider, mw = self.get_request_spider_mw() - mw.spider_opened(spider) - assert mw.process_request(req) is None - assert req.meta.get("download_timeout") == 180 - def test_string_download_timeout(self): - req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) - mw.spider_opened(spider) - assert mw.process_request(req) is None - assert req.meta.get("download_timeout") == 20.1 +def test_default_download_timeout(): + req, spider, mw = get_request_spider_mw() + mw.spider_opened(spider) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 180 - def test_setting_has_download_timeout(self): - req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2}) - mw.spider_opened(spider) - assert mw.process_request(req) is None - assert req.meta.get("download_timeout") == 2 - def test_request_has_download_timeout(self): - req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2}) - mw.spider_opened(spider) - req.meta["download_timeout"] = 1 - assert mw.process_request(req) is None - assert req.meta.get("download_timeout") == 1 +def test_string_download_timeout(): + req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) + mw.spider_opened(spider) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 20.1 - def test_zero_download_timeout(self): - req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0}) - mw.spider_opened(spider) - assert mw.process_request(req) is None - assert req.meta.get("download_timeout") is None + +def test_setting_has_download_timeout(): + req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2}) + mw.spider_opened(spider) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 2 + + +def test_request_has_download_timeout(): + req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2}) + mw.spider_opened(spider) + req.meta["download_timeout"] = 1 + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") == 1 + + +def test_zero_download_timeout(): + req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0}) + mw.spider_opened(spider) + assert mw.process_request(req) is None + assert req.meta.get("download_timeout") is None diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 539183cd6..0c925a0d1 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,33 +1,37 @@ +from __future__ import annotations + from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestUserAgentMiddleware: - def get_spider_and_mw(self, default_useragent): - crawler = get_crawler(Spider, {"USER_AGENT": default_useragent}) - spider = crawler._create_spider("foo") - return spider, UserAgentMiddleware.from_crawler(crawler) +def get_spider_and_mw( + default_useragent: str | None, +) -> tuple[Spider, UserAgentMiddleware]: + crawler = get_crawler(Spider, {"USER_AGENT": default_useragent}) + spider = crawler._create_spider("foo") + return spider, UserAgentMiddleware.from_crawler(crawler) - def test_default_agent(self): - _, mw = self.get_spider_and_mw("default_useragent") - req = Request("http://scrapytest.org/") - assert mw.process_request(req) is None - assert req.headers["User-Agent"] == b"default_useragent" - def test_header_agent(self): - spider, mw = self.get_spider_and_mw("default_useragent") - mw.spider_opened(spider) - req = Request( - "http://scrapytest.org/", headers={"User-Agent": "header_useragent"} - ) - assert mw.process_request(req) is None - assert req.headers["User-Agent"] == b"header_useragent" +def test_default_agent(): + _, mw = get_spider_and_mw("default_useragent") + req = Request("http://scrapytest.org/") + assert mw.process_request(req) is None + assert req.headers["User-Agent"] == b"default_useragent" - def test_no_agent(self): - spider, mw = self.get_spider_and_mw(None) - mw.spider_opened(spider) - req = Request("http://scrapytest.org/") - assert mw.process_request(req) is None - assert "User-Agent" not in req.headers + +def test_header_agent(): + spider, mw = get_spider_and_mw("default_useragent") + mw.spider_opened(spider) + req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"}) + assert mw.process_request(req) is None + assert req.headers["User-Agent"] == b"header_useragent" + + +def test_no_agent(): + spider, mw = get_spider_and_mw(None) + mw.spider_opened(spider) + req = Request("http://scrapytest.org/") + assert mw.process_request(req) is None + assert "User-Agent" not in req.headers diff --git a/tests/test_engine.py b/tests/test_engine.py index e51eb4664..2b857d35a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -38,6 +38,8 @@ from tests import get_testdata from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: + from collections.abc import AsyncIterator + from twisted.python.failure import Failure from scrapy.core.scheduler import Scheduler @@ -608,8 +610,8 @@ class TestEngineDownload(TestEngineDownloadAsync): @coroutine_test async def test_request_scheduled_signal(): class TestScheduler(BaseScheduler): - def __init__(self): - self.enqueued = [] + def __init__(self) -> None: + self.enqueued: list[Request] = [] def enqueue_request(self, request: Request) -> bool: self.enqueued.append(request) @@ -621,9 +623,9 @@ async def test_request_scheduled_signal(): crawler = get_crawler(MySpider) engine = ExecutionEngine(crawler, lambda _: None) - scheduler = TestScheduler() + scheduler = TestScheduler() # type: ignore[abstract] - async def start(): + async def start() -> AsyncIterator[Any]: return yield diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 18782fb70..0fe2caca4 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -102,7 +102,9 @@ class TestPeriodicLog: ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict[str, Any], condition: Callable) -> None: + def check( + settings: dict[str, Any], condition: Callable[[str, Any], bool] + ) -> None: ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -168,7 +170,9 @@ class TestPeriodicLog: ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict[str, Any], condition: Callable) -> None: + def check( + settings: dict[str, Any], condition: Callable[[str, Any], bool] + ) -> None: ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 0a926479b..469985288 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -378,6 +378,7 @@ class TestBatchDeliveries(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(total=2, mockserver=self.mockserver) + assert crawler.stats assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @@ -392,7 +393,9 @@ class TestBatchDeliveries(TestFeedExportBase): ] class CustomS3FeedStorage(S3FeedStorage): - stubs = [] + from botocore.stub import Stubber # noqa: PLC0415 + + stubs: list[Stubber] = [] def open(self, *args, **kwargs): from botocore import __version__ as botocore_version # noqa: PLC0415 @@ -449,6 +452,7 @@ class TestBatchDeliveries(TestFeedExportBase): assert len(CustomS3FeedStorage.stubs) == len(items) for stub in CustomS3FeedStorage.stubs: stub.assert_no_pending_responses() + assert crawler.stats assert ( "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() ) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index ce9296764..5d4d5a2bd 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from http.cookiejar import DefaultCookiePolicy from scrapy.http import Request, Response @@ -46,7 +48,7 @@ class TestCookieJar: def test_set_policy(self): policy = DefaultCookiePolicy() self.jar.set_policy(policy) - assert self.jar.jar._policy is policy + assert self.jar.jar._policy is policy # type: ignore[attr-defined] def test_check_expired_frequency(self): jar = CookieJar(check_expired_frequency=1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 42ab29267..7c9f717e4 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,3 +1,5 @@ +from typing import Any + from scrapy.http import ( Headers, HtmlResponse, @@ -38,13 +40,13 @@ class TestResponseTypes: ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) - assert retcls is cls, f"{source} ==> {retcls} != {cls}" + assert retcls is cls, f"{source!r} ==> {retcls} != {cls}" def test_from_content_disposition_no_filename(self): assert responsetypes.from_content_disposition(b"attachment") is Response def test_from_content_type(self): - mappings = [ + mappings: list[tuple[str | bytes, type[Response]]] = [ ("text/html; charset=UTF-8", HtmlResponse), ("text/xml; charset=UTF-8", XmlResponse), ("application/xhtml+xml; charset=UTF-8", HtmlResponse), @@ -58,7 +60,7 @@ class TestResponseTypes: ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) - assert retcls is cls, f"{source} ==> {retcls} != {cls}" + assert retcls is cls, f"{source!r} ==> {retcls} != {cls}" def test_from_body(self): mappings = [ @@ -71,7 +73,7 @@ class TestResponseTypes: ] for source, cls in mappings: retcls = responsetypes.from_body(source) - assert retcls is cls, f"{source} ==> {retcls} != {cls}" + assert retcls is cls, f"{source!r} ==> {retcls} != {cls}" def test_from_headers(self): mappings = [ @@ -98,7 +100,7 @@ class TestResponseTypes: def test_from_args(self): # TODO: add more tests that check precedence between the different arguments - mappings = [ + mappings: list[tuple[dict[str, Any], type[Response]]] = [ ({"url": "http://www.example.com/data.csv"}, TextResponse), # headers takes precedence over url ( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8637de41e..39ce87b65 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -379,13 +379,14 @@ class TestIntegrationWithDownloaderAwareInMemory: url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) + assert self.crawler.stats assert self.crawler.stats.get_value("downloader/response_count") == len( start_urls ) class TestIncompatibility: - def _incompatible(self): + def _incompatible(self) -> None: settings = { "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", "CONCURRENT_REQUESTS_PER_IP": 1, diff --git a/tests/test_signals.py b/tests/test_signals.py index f5f13b992..fc9520f49 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,13 +1,20 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest from scrapy import Request, Spider, signals from scrapy.utils.test import get_crawler, get_from_asyncio_queue -from tests.mockserver.http import MockServer -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class ItemSpider(Spider): name = "itemspider" + mockserver: MockServer async def start(self): for index in range(10): @@ -34,15 +41,6 @@ class TestMain: class TestMockServer: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - def setup_method(self): self.items = [] @@ -51,11 +49,11 @@ class TestMockServer: self.items.append(item) @pytest.mark.only_asyncio - @inline_callbacks_test - def test_simple_pipeline(self): + @coroutine_test + async def test_simple_pipeline(self, mockserver: MockServer) -> None: crawler = get_crawler(ItemSpider) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) assert len(self.items) == 10 for index in range(10): assert {"index": index} in self.items diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 9257e0232..7cef2fefa 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,7 +1,7 @@ from __future__ import annotations from asyncio import sleep -from typing import Any +from typing import TYPE_CHECKING, Any import pytest @@ -12,6 +12,9 @@ from scrapy.utils.test import get_crawler from .utils import twisted_sleep from .utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + SLEEP_SECONDS = 0.1 ITEM_A = {"id": "a"} @@ -67,7 +70,11 @@ class TestMain: await self._test_spider(TestSpider, [ITEM_A]) - async def _test_start(self, start_, expected_items=None): + async def _test_start( + self, + start_: Callable[[Any], AsyncIterator[Any]], + expected_items: list[Any] | None = None, + ) -> None: class TestSpider(Spider): name = "test" start = start_ diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index e5891474b..7824d5c20 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -186,7 +186,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware: class ProcessSpiderExceptionAsyncIteratorMiddleware: async def process_spider_exception(self, response, exception): yield {"foo": 1} - d = defer.Deferred() + d: defer.Deferred[None] = defer.Deferred() call_later(0, d.callback, None) await maybe_deferred_to_future(d) yield {"foo": 2} diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 6a054ded5..2ba083fd6 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -22,6 +22,7 @@ class _HttpErrorSpider(MockServerSpider): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + assert self.mockserver self.start_urls = [ self.mockserver.url("/status?n=200"), self.mockserver.url("/status?n=404"), @@ -36,7 +37,7 @@ class _HttpErrorSpider(MockServerSpider): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) - def parse(self, response): + def parse(self, response: Response) -> None: self.parsed.add(response.url[-3:]) def on_error(self, failure): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 32e83f990..7591c0e7b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse import pytest @@ -35,6 +35,9 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncIterator + class TestRefererMiddleware: req_meta: dict[str, Any] = {} @@ -1031,7 +1034,7 @@ async def test_response_policy_only_supports_policy_names(): crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"}) mw = build_from_crawler(RefererMiddleware, crawler) - async def input_result(): + async def input_result() -> AsyncIterator[Any]: yield Request("https://example.com/") response = Response( @@ -1079,7 +1082,7 @@ async def test_referer_policies_setting(): ) mw = build_from_crawler(RefererMiddleware, crawler) - async def input_result(): + async def input_result() -> AsyncIterator[Any]: yield Request("https://example.com/") # "no-referrer-when-downgrade": None, diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index def3a3df3..e879f3a32 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.http import Request, Response from scrapy.spidermiddlewares.start import StartSpiderMiddleware from scrapy.spiders import Spider @@ -5,6 +9,9 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncIterator + class TestMiddleware: @coroutine_test @@ -12,7 +19,7 @@ class TestMiddleware: crawler = get_crawler(Spider) mw = build_from_crawler(StartSpiderMiddleware, crawler) - async def start(): + async def start() -> AsyncIterator[Request]: yield Request("data:,1") yield Request("data:,2", meta={"is_start_request": True}) yield Request("data:,2", meta={"is_start_request": False}) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a198d1c09..7528dc51a 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -72,7 +72,7 @@ class TestParallelAsyncio: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable(length) await _parallel_asyncio( ait, @@ -91,7 +91,7 @@ class TestParallelAsyncio: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable_with_delays(length) await _parallel_asyncio( ait, diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index b6a9d8b06..518cc3518 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import Any + import pytest from scrapy.exceptions import UsageError @@ -106,7 +110,7 @@ class TestFeedExportConfig: ) def test_feed_complete_default_values_from_settings_empty(self): - feed = {} + feed: dict[str, Any] = {} settings = Settings( { "FEED_EXPORT_ENCODING": "custom encoding", diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index dc1d96f66..05b272832 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.utils.console import get_shell_embed_func @@ -38,4 +40,5 @@ def test_get_shell_embed_func_bpython(): def test_get_shell_embed_func_ipython(): # default shell should be 'ipython' shell = get_shell_embed_func() + assert shell is not None assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 8b02116b8..9f9e24957 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,9 +1,16 @@ +from __future__ import annotations + import builtins from io import StringIO +from typing import TYPE_CHECKING from unittest import mock from scrapy.utils.display import pformat, pprint +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + from types import ModuleType + value = {"a": 1} colorized_strings = { ( @@ -77,14 +84,19 @@ def test_pformat_no_pygments(isatty): real_import = builtins.__import__ - def mock_import(name, globals_, locals_, fromlist, level): + def mock_import( + name: str, + globals_: Mapping[str, object] | None = None, + locals_: Mapping[str, object] | None = None, + fromlist: Sequence[str] | None = (), + level: int = 0, + ) -> ModuleType: if "pygments" in name: raise ImportError return real_import(name, globals_, locals_, fromlist, level) - builtins.__import__ = mock_import - assert pformat(value) == plain_string - builtins.__import__ = real_import + with mock.patch("builtins.__import__", mock_import): + assert pformat(value) == plain_string def test_pprint(): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 1276c1392..a59541edb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -238,7 +238,7 @@ class TestXmliterBase(ABC): next(my_iter) def test_xmliter_objtype_exception(self): - i = self.xmliter(42, "product") + i = self.xmliter(42, "product") # type: ignore[arg-type] with pytest.raises(TypeError): next(i) @@ -353,7 +353,7 @@ class TestLxmlXmliter(TestXmliterBase): assert node.xpath("f:name/text()").getall() == ["African Coffee Table"] def test_xmliter_objtype_exception(self): - i = self.xmliter(42, "product") + i = self.xmliter(42, "product") # type: ignore[arg-type] with pytest.raises(TypeError): next(i) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index ee552df64..fe57492b6 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -38,7 +38,7 @@ class TestFailureToExcInfo: assert exc_info == failure_to_exc_info(failure) def test_non_failure(self): - assert failure_to_exc_info("test") is None + assert failure_to_exc_info("test") is None # type: ignore[arg-type] class TestTopLevelFormatter: diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index c4c861404..ab6965ed5 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import sys from pathlib import Path @@ -37,7 +39,7 @@ class TestUtilsMisc: with pytest.raises(NameError): load_object("scrapy.utils.misc.load_object999") with pytest.raises(TypeError): - load_object({}) + load_object({}) # type: ignore[arg-type] def test_walk_modules(self): mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") @@ -114,7 +116,7 @@ class TestUtilsMisc: args = (True, 100.0) kwargs = {"key": "val"} - def _test_with_crawler(mock, crawler): + def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: build_from_crawler(mock, crawler, *args, **kwargs) if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 4544cd29e..146cdb802 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -38,7 +38,7 @@ def test_open_in_browser(): resp = Response(url, body=body) with pytest.raises(TypeError): - open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg + open_in_browser(resp, _openfunc=browser_open) # type: ignore[arg-type] def test_get_meta_refresh(): @@ -320,4 +320,4 @@ def test_open_in_browser_text_response_uses_txt_extension(): def test_open_in_browser_raises_for_unsupported_response_type(): response = Response("http://www.example.com", body=b"binary") with pytest.raises(TypeError): - open_in_browser(response, _openfunc=lambda _: True) + open_in_browser(response, _openfunc=lambda _: True) # type: ignore[arg-type] diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 9ac0fd0c0..dc615eca6 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import asyncio +from typing import TYPE_CHECKING import pytest from pydispatch import dispatcher @@ -16,6 +19,9 @@ from scrapy.utils.signal import ( from scrapy.utils.test import get_from_asyncio_queue from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from collections.abc import Callable + class TestSendCatchLog: # whether the function being tested returns exceptions or failures @@ -24,7 +30,7 @@ class TestSendCatchLog: @inline_callbacks_test def test_send_catch_log(self): test_signal = object() - handlers_called = set() + handlers_called: set[Callable[..., None]] = set() dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) @@ -74,7 +80,7 @@ class TestSendCatchLogDeferred2(TestSendCatchLogDeferred): def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" - d = defer.Deferred() + d: defer.Deferred[str] = defer.Deferred() call_later(0, d.callback, "OK") return d @@ -108,7 +114,7 @@ class TestSendCatchLogAsync2(TestSendCatchLogAsync): def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" - d = defer.Deferred() + d: defer.Deferred[str] = defer.Deferred() call_later(0, d.callback, "OK") return d diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 05971f423..71f0f76ad 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from scrapy import Spider from scrapy.http import Request from scrapy.item import Item @@ -17,7 +19,7 @@ def test_iterate_spider_output(): r = Request("http://scrapytest.org") o = object() - assert list(iterate_spider_output(i)) == [i] + assert list(iterate_spider_output(i)) == [i] # type: ignore[call-overload] assert list(iterate_spider_output(r)) == [r] assert list(iterate_spider_output(o)) == [o] assert list(iterate_spider_output([r, i, o])) == [r, i, o] diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 7a7a264d3..2334c76e9 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -79,9 +79,9 @@ def test_get_oldest(): stale entries in `live_refs`, affecting results unless explicitly cleared. """ - def _delete_o1(): + def _delete_o1() -> None: """Delete `o1` and ensure it is actually collected on PyPy.""" - nonlocal o1 + nonlocal o1 # type: ignore[misc] del o1 if _IS_PYPY: @@ -89,7 +89,7 @@ def test_get_oldest(): # still exist until the GC runs, so we force a collection cycle. garbage_collect() - def _do_asserts(): + def _do_asserts() -> None: assert trackref.get_oldest("Foo") is o1 assert trackref.get_oldest("Bar") is o2 # Ensure the newer Foo is not incorrectly considered the oldest