From b4a8c1145386a6eccda47e83da5c6ec8cfd47d88 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 1 Feb 2025 23:16:12 +0500 Subject: [PATCH] Make download handler test base classes abstract. --- tests/test_downloader_handlers.py | 41 ++++++++++++++++++------- tests/test_downloader_handlers_http2.py | 19 ++++++------ 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ae2030fe6..d816e3aed 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,6 +4,7 @@ import contextlib import os import shutil import sys +from abc import ABC, abstractmethod from pathlib import Path from tempfile import mkdtemp, mkstemp from unittest import SkipTest, mock @@ -19,10 +20,9 @@ from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss from w3lib.url import path_to_file_uri -from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers import DownloadHandlerProtocol, DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler @@ -218,14 +218,18 @@ class DuplicateHeaderResource(resource.Resource): return b"" -class HttpTestCase(unittest.TestCase): +class HttpTestCase(unittest.TestCase, ABC): scheme = "http" - download_handler_cls: type = HTTPDownloadHandler # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + def setUp(self): self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") @@ -426,7 +430,9 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls: type = HTTP10DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -443,7 +449,9 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls: type = HTTP11DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) @@ -644,11 +652,14 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = "https" - download_handler_cls: type = HTTP11DownloadHandler keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler + def setUp(self): self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") @@ -738,10 +749,14 @@ class UriResource(resource.Resource): return b"" -class HttpProxyTestCase(unittest.TestCase): - download_handler_cls: type = HTTPDownloadHandler +class HttpProxyTestCase(unittest.TestCase, ABC): expected_http_proxy_request_body = b"http://example.com" + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + def setUp(self): site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) @@ -783,11 +798,15 @@ class HttpProxyTestCase(unittest.TestCase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls: type = HTTP10DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls: type = HTTP11DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 322075043..cc5df7b60 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -9,6 +9,7 @@ from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED +from scrapy.core.downloader.handlers import DownloadHandlerProtocol from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler @@ -28,11 +29,11 @@ class Https2TestCase(Https11TestCase): scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - @classmethod - def setUpClass(cls): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - cls.download_handler_cls = H2DownloadHandler + return H2DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -197,11 +198,11 @@ class Https2InvalidDNSPattern(Https2TestCase): class Https2CustomCiphers(Https11CustomCiphers): scheme = "https" - @classmethod - def setUpClass(cls): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - cls.download_handler_cls = H2DownloadHandler + return H2DownloadHandler class Http2MockServerTestCase(Http11MockServerTestCase): @@ -225,11 +226,11 @@ class Https2ProxyTestCase(Http11ProxyTestCase): expected_http_proxy_request_body = b"/" - @classmethod - def setUpClass(cls): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - cls.download_handler_cls = H2DownloadHandler + return H2DownloadHandler def setUp(self): site = server.Site(UriResource(), timeout=None)