From b4d11b8b2565b5686c9f395ca1a8dd085609aafc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Jun 2025 04:28:09 +0500 Subject: [PATCH] Further reduce deps on unittest. (#6884) --- .../test_downloader_handler_twisted_http11.py | 10 +- .../test_downloader_handler_twisted_http2.py | 14 +- tests/test_downloader_handlers.py | 7 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine_loop.py | 4 +- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 43 +- tests/test_http2_client_protocol.py | 480 ++++++++---------- tests/test_pipeline_crawl.py | 15 +- tests/test_pipeline_files.py | 16 +- tests/test_pipeline_images.py | 17 +- tests/test_pipeline_media.py | 2 +- tests/test_scheduler_base.py | 6 +- tests/test_signals.py | 4 +- tests/test_spider_start.py | 2 +- tests/test_spidermiddleware_process_start.py | 4 +- tests/test_utils_defer.py | 6 +- tests/test_utils_signal.py | 16 +- tests/test_webclient.py | 89 ++-- 20 files changed, 355 insertions(+), 388 deletions(-) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 70f55e787..7b26ce03f 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -39,23 +39,21 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): pass -class Https11WrongHostnameTestCase( - HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase -): +class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https11InvalidDNSPattern( +class TestHttps11InvalidDNSPattern( HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index e058cedae..3e685bb28 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -163,23 +163,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): assert json.loads(response.text)["headers"][header] == [value1, value2] -class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase): +class TestHttps2InvalidDNSPattern( + H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): pass -class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class Http2MockServerTestCase(TestHttpMockServerBase): +class TestHttp2MockServer(TestHttpMockServerBase): """HTTP 2.0 test case with MockServer""" @property @@ -193,7 +195,7 @@ class Http2MockServerTestCase(TestHttpMockServerBase): is_secure = True -class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): +class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2c8e96040..518dc6b24 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -12,6 +12,7 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal +from twisted.internet.defer import inlineCallbacks from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -340,9 +341,10 @@ class TestFTPBase(unittest.TestCase): self.portNum = self.port.getHost().port crawler = get_crawler() self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + @inlineCallbacks def tearDown(self): + yield self.port.stopListening() shutil.rmtree(self.directory) def _add_test_callbacks(self, deferred, callback=None, errback=None): @@ -478,9 +480,10 @@ class TestAnonymousFTP(TestFTPBase): self.portNum = self.port.getHost().port crawler = get_crawler() self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + @inlineCallbacks def tearDown(self): + yield self.port.stopListening() shutil.rmtree(self.directory) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 146b0057e..dd5d47cab 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -247,10 +247,8 @@ Disallow: /some/randome/page.html assert request.callback == NO_CALLBACK +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): - if not rerp_available(): - skip = "Rerp parser is not installed" - def setUp(self): super().setUp() self.crawler.settings.set( diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9b7c09448..ddac95edf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -49,7 +49,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): self.times[slot].append(time.time()) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c7dbc82d4..bfb8eeced 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -27,7 +27,7 @@ async def sleep(seconds: float = 0.001) -> None: await maybe_deferred_to_future(deferred) -class MainTestCase(TestCase): +class TestMain(TestCase): @deferred_f_from_coro_f async def test_sleep(self): """Neither asynchronous sleeps on Spider.start() nor the equivalent on @@ -119,7 +119,7 @@ class MainTestCase(TestCase): assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" -class RequestSendOrderTestCase(TestCase): +class TestRequestSendOrder(TestCase): seconds = 0.1 # increase if flaky @classmethod diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 2ac4d7830..f9e54cb28 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -8,7 +8,7 @@ from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TestTelnetExtension(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 262c0b434..01797fd20 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -91,52 +91,53 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -# TODO: replace self.mktemp() and drop the unittest.TestCase base -class TestFileFeedStorage(unittest.TestCase): - def test_store_file_uri(self): - path = Path(self.mktemp()).resolve() +class TestFileFeedStorage: + def test_store_file_uri(self, tmp_path): + path = tmp_path / "file.txt" uri = path_to_file_uri(str(path)) self._assert_stores(FileFeedStorage(uri), path) - def test_store_file_uri_makedirs(self): - path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" + def test_store_file_uri_makedirs(self, tmp_path): + path = tmp_path / "more" / "paths" / "file.txt" uri = path_to_file_uri(str(path)) self._assert_stores(FileFeedStorage(uri), path) - def test_store_direct_path(self): - path = Path(self.mktemp()).resolve() + def test_store_direct_path(self, tmp_path): + path = tmp_path / "file.txt" self._assert_stores(FileFeedStorage(str(path)), path) - def test_store_direct_path_relative(self): - path = Path(self.mktemp()) + def test_store_direct_path_relative(self, tmp_path): + path = (tmp_path / "foo" / "bar").relative_to(Path.cwd()) self._assert_stores(FileFeedStorage(str(path)), path) - def test_interface(self): - path = self.mktemp() - st = FileFeedStorage(path) + def test_interface(self, tmp_path): + path = tmp_path / "file.txt" + st = FileFeedStorage(str(path)) verifyObject(IFeedStorage, st) - def _store(self, feed_options=None) -> Path: - path = Path(self.mktemp()).resolve() + @staticmethod + def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: storage = FileFeedStorage(str(path), feed_options=feed_options) spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") storage.store(file) - return path - def test_append(self): - path = self._store() + def test_append(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path) self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") - def test_overwrite(self): - path = self._store({"overwrite": True}) + def test_overwrite(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path, {"overwrite": True}) self._assert_stores( FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) + @staticmethod def _assert_stores( - self, storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" ) -> None: spider = scrapy.Spider("default") file = storage.open(spider) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ef1806cc0..80edd50d6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -8,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable from unittest import mock from urllib.parse import urlencode @@ -32,17 +32,22 @@ from twisted.web.static import File from scrapy.http import JsonRequest, Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from tests.mockserver import LeafResource, Status, ssl_context_factory if TYPE_CHECKING: - from twisted.python.failure import Failure + from collections.abc import Coroutine -def generate_random_string(size): +def generate_random_string(size: int) -> str: return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) -def make_html_body(val): +def make_html_body(val: str) -> bytes: response = f"""

Hello from HTTP2

{val}

@@ -92,7 +97,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod - def make_response(request: TxRequest, extra_data: str): + def make_response(request: TxRequest, extra_data: str) -> bytes: assert request.content is not None response = { "request-headers": {}, @@ -179,7 +184,6 @@ def get_client_certificate( pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text( encoding="utf-8" ) - return PrivateCertificate.loadPEM(pem) @@ -238,6 +242,7 @@ class TestHttps2ClientProtocol(TestCase): uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8")) self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) @@ -255,7 +260,7 @@ class TestHttps2ClientProtocol(TestCase): shutil.rmtree(self.temp_directory) self.conn_closed_deferred = None - def get_url(self, path): + def get_url(self, path: str) -> str: """ :param path: Should have / at the starting compulsorily if not empty :return: Complete url @@ -264,143 +269,146 @@ class TestHttps2ClientProtocol(TestCase): assert path[0] == "/" or path[0] == "&" return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" - def make_request(self, request: Request) -> Deferred: + async def make_request(self, request: Request) -> Response: + return await maybe_deferred_to_future(self.make_request_dfd(request)) + + def make_request_dfd(self, request: Request) -> Deferred[Response]: return self.client.request(request, DummySpider()) @staticmethod - def _check_repeat(get_deferred, count): + async def _check_repeat( + get_coro: Callable[[], Coroutine[Any, Any, None]], count: int + ) -> None: d_list = [] for _ in range(count): - d = get_deferred() + d = deferred_from_coro(get_coro()) d_list.append(d) - return DeferredList(d_list, fireOnOneErrback=True) + await maybe_deferred_to_future(DeferredList(d_list, fireOnOneErrback=True)) - def _check_GET(self, request: Request, expected_body, expected_status): - def check_response(response: Response): - assert response.status == expected_status - assert response.body == expected_body - assert response.request == request + async def _check_GET( + self, request: Request, expected_body: bytes, expected_status: int + ) -> None: + response = await self.make_request(request) + assert response.status == expected_status + assert response.body == expected_body + assert response.request == request - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - assert len(response.body) == content_length + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - d = self.make_request(request) - d.addCallback(check_response) - d.addErrback(self.fail) - return d - - def test_GET_small_body(self): + @deferred_f_from_coro_f + async def test_GET_small_body(self): request = Request(self.get_url("/get-data-html-small")) - return self._check_GET(request, Data.HTML_SMALL, 200) + await self._check_GET(request, Data.HTML_SMALL, 200) - def test_GET_large_body(self): + @deferred_f_from_coro_f + async def test_GET_large_body(self): request = Request(self.get_url("/get-data-html-large")) - return self._check_GET(request, Data.HTML_LARGE, 200) + await self._check_GET(request, Data.HTML_LARGE, 200) - def _check_GET_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_GET(*args, **kwargs) + async def _check_GET_x10( + self, request: Request, expected_body: bytes, expected_status: int + ) -> None: + async def get_coro() -> None: + await self._check_GET(request, expected_body, expected_status) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_GET_small_body_x10(self): - return self._check_GET_x10( + @deferred_f_from_coro_f + async def test_GET_small_body_x10(self): + await self._check_GET_x10( Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) - def test_GET_large_body_x10(self): - return self._check_GET_x10( + @deferred_f_from_coro_f + async def test_GET_large_body_x10(self): + await self._check_GET_x10( Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200 ) - def _check_POST_json( + async def _check_POST_json( self, request: Request, - expected_request_body, - expected_extra_data, + expected_request_body: dict[str, str], + expected_extra_data: str, expected_status: int, - ): - d = self.make_request(request) + ) -> None: + response = await self.make_request(request) - def assert_response(response: Response): - assert response.status == expected_status - assert response.request == request + assert response.status == expected_status + assert response.request == request - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - assert len(response.body) == content_length + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - # Parse the body - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - body = json.loads(str(response.body, content_encoding)) - assert "request-body" in body - assert "extra-data" in body - assert "request-headers" in body + # Parse the body + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + body = json.loads(str(response.body, content_encoding)) + assert "request-body" in body + assert "extra-data" in body + assert "request-headers" in body - request_body = body["request-body"] - assert request_body == expected_request_body + request_body = body["request-body"] + assert request_body == expected_request_body - extra_data = body["extra-data"] - assert extra_data == expected_extra_data + extra_data = body["extra-data"] + assert extra_data == expected_extra_data - # Check if headers were sent successfully - request_headers = body["request-headers"] - for k, v in request.headers.items(): - k_str = str(k, "utf-8") - assert k_str in request_headers - assert request_headers[k_str] == str(v[0], "utf-8") + # Check if headers were sent successfully + request_headers = body["request-headers"] + for k, v in request.headers.items(): + k_str = str(k, "utf-8") + assert k_str in request_headers + assert request_headers[k_str] == str(v[0], "utf-8") - d.addCallback(assert_response) - d.addErrback(self.fail) - return d - - def test_POST_small_json(self): + @deferred_f_from_coro_f + async def test_POST_small_json(self): request = JsonRequest( url=self.get_url("/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) + await self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) - def test_POST_large_json(self): + @deferred_f_from_coro_f + async def test_POST_large_json(self): request = JsonRequest( url=self.get_url("/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) + await self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) - def _check_POST_json_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_POST_json(*args, **kwargs) + async def _check_POST_json_x10(self, *args, **kwargs): + async def get_coro() -> None: + await self._check_POST_json(*args, **kwargs) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_POST_small_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_small_json_x10(self): request = JsonRequest( url=self.get_url("/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json_x10( - request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 - ) + await self._check_POST_json_x10(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) - def test_POST_large_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_large_json_x10(self): request = JsonRequest( url=self.get_url("/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json_x10( - request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 - ) + await self._check_POST_json_x10(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) @inlineCallbacks def test_invalid_negotiated_protocol(self): @@ -409,77 +417,59 @@ class TestHttps2ClientProtocol(TestCase): ): request = Request(url=self.get_url("/status?n=200")) with pytest.raises(ResponseFailed): - yield self.make_request(request) + yield self.make_request_dfd(request) + @inlineCallbacks def test_cancel_request(self): request = Request(url=self.get_url("/get-data-html-large")) - - def assert_response(response: Response): - assert response.status == 499 - assert response.request == request - - d = self.make_request(request) - d.addCallback(assert_response) - d.addErrback(self.fail) + d = self.make_request_dfd(request) d.cancel() + response = yield d + assert response.status == 499 + assert response.request == request - return d - - def test_download_maxsize_exceeded(self): + @deferred_f_from_coro_f + async def test_download_maxsize_exceeded(self): request = Request( url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000} ) + with pytest.raises(CancelledError) as exc_info: + await self.make_request(request) + error_pattern = re.compile( + rf"Cancelling download of {request.url}: received response " + rf"size \(\d*\) larger than download max size \(1000\)" + ) + assert len(re.findall(error_pattern, str(exc_info.value))) == 1 - def assert_cancelled_error(failure): - assert isinstance(failure.value, CancelledError) - error_pattern = re.compile( - rf"Cancelling download of {request.url}: received response " - rf"size \(\d*\) larger than download max size \(1000\)" - ) - assert len(re.findall(error_pattern, str(failure.value))) == 1 - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_cancelled_error) - return d - + @inlineCallbacks def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" + from h2.exceptions import InvalidBodyLengthError + request = Request(url=self.get_url("/dataloss")) + with pytest.raises(ResponseFailed) as exc_info: + yield self.make_request_dfd(request) + assert len(exc_info.value.reasons) > 0 + assert any( + isinstance(error, InvalidBodyLengthError) + for error in exc_info.value.reasons + ) - def assert_failure(failure: Failure): - assert len(failure.value.reasons) > 0 - from h2.exceptions import InvalidBodyLengthError - - assert any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - ) - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_failure) - return d - - def test_missing_content_length_header(self): + @deferred_f_from_coro_f + async def test_missing_content_length_header(self): request = Request(url=self.get_url("/no-content-length-header")) + response = await self.make_request(request) + assert response.status == 200 + assert response.body == Data.NO_CONTENT_LENGTH + assert response.request == request + assert "Content-Length" not in response.headers - def assert_content_length(response: Response): - assert response.status == 200 - assert response.body == Data.NO_CONTENT_LENGTH - assert response.request == request - assert "Content-Length" not in response.headers - - d = self.make_request(request) - d.addCallback(assert_content_length) - d.addErrback(self.fail) - return d - - @inlineCallbacks - def _check_log_warnsize(self, request, warn_pattern, expected_body): + async def _check_log_warnsize( + self, request: Request, warn_pattern: re.Pattern[str], expected_body: bytes + ) -> None: with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: - response = yield self.make_request(request) + response = await self.make_request(request) assert response.status == 200 assert response.request == request assert response.body == expected_body @@ -487,8 +477,8 @@ class TestHttps2ClientProtocol(TestCase): # Check the warning is raised only once for this request assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1 - @inlineCallbacks - def test_log_expected_warnsize(self): + @deferred_f_from_coro_f + async def test_log_expected_warnsize(self): request = Request( url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000} ) @@ -497,10 +487,10 @@ class TestHttps2ClientProtocol(TestCase): rf"download warn size \(1000\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + await self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) - @inlineCallbacks - def test_log_received_warnsize(self): + @deferred_f_from_coro_f + async def test_log_received_warnsize(self): request = Request( url=self.get_url("/no-content-length-header"), meta={"download_warnsize": 10}, @@ -510,20 +500,22 @@ class TestHttps2ClientProtocol(TestCase): rf"warn size \(10\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + await self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) - def test_max_concurrent_streams(self): + @deferred_f_from_coro_f + async def test_max_concurrent_streams(self): """Send 500 requests at one to check if we can handle very large number of request. """ - def get_deferred(): - return self._check_GET( + async def get_coro() -> None: + await self._check_GET( Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) - return self._check_repeat(get_deferred, 500) + await self._check_repeat(get_coro, 500) + @inlineCallbacks def test_inactive_stream(self): """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS by default is 100. After sending the first 100 requests we close the @@ -532,6 +524,7 @@ class TestHttps2ClientProtocol(TestCase): def assert_inactive_stream(failure): assert failure.check(ResponseFailed) is not None + from scrapy.core.http2.stream import InactiveStreamClosed assert any( @@ -540,14 +533,14 @@ class TestHttps2ClientProtocol(TestCase): # Send 100 request (we do not check the result) for _ in range(100): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) + d = self.make_request_dfd(Request(self.get_url("/get-data-html-small"))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) - d.addCallback(self.fail) + d = self.make_request_dfd(Request(self.get_url("/get-data-html-small"))) + d.addCallback(lambda _: pytest.fail("This request should have failed")) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -555,13 +548,15 @@ class TestHttps2ClientProtocol(TestCase): # with InactiveStreamClosed self.client.transport.loseConnection() - return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + yield DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) - def test_invalid_request_type(self): + @deferred_f_from_coro_f + async def test_invalid_request_type(self): with pytest.raises(TypeError): - self.make_request("https://InvalidDataTypePassed.com") + await self.make_request("https://InvalidDataTypePassed.com") - def test_query_parameters(self): + @deferred_f_from_coro_f + async def test_query_parameters(self): params = { "a": generate_random_string(20), "b": generate_random_string(20), @@ -569,133 +564,96 @@ class TestHttps2ClientProtocol(TestCase): "d": generate_random_string(20), } request = Request(self.get_url(f"/query-params?{urlencode(params)}")) + response = await self.make_request(request) + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + data = json.loads(str(response.body, content_encoding)) + assert data == params - def assert_query_params(response: Response): - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - data = json.loads(str(response.body, content_encoding)) - assert data == params - - d = self.make_request(request) - d.addCallback(assert_query_params) - d.addErrback(self.fail) - - return d - - def test_status_codes(self): - def assert_response_status(response: Response, expected_status: int): - assert response.status == expected_status - - d_list = [] + @deferred_f_from_coro_f + async def test_status_codes(self): for status in [200, 404]: request = Request(self.get_url(f"/status?n={status}")) - d = self.make_request(request) - d.addCallback(assert_response_status, status) - d.addErrback(self.fail) - d_list.append(d) + response = await self.make_request(request) + assert response.status == status - return DeferredList(d_list, fireOnOneErrback=True) - - def test_response_has_correct_certificate_ip_address(self): + @deferred_f_from_coro_f + async def test_response_has_correct_certificate_ip_address(self): request = Request(self.get_url("/status?n=200")) + response = await self.make_request(request) + assert response.request == request + assert isinstance(response.certificate, Certificate) + assert response.certificate.original is not None + assert response.certificate.getIssuer() == self.client_certificate.getIssuer() + assert response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() + ) + assert isinstance(response.ip_address, IPv4Address) + assert str(response.ip_address) == "127.0.0.1" - def assert_metadata(response: Response): - assert response.request == request - assert isinstance(response.certificate, Certificate) - assert response.certificate.original is not None - assert ( - response.certificate.getIssuer() == self.client_certificate.getIssuer() - ) - assert response.certificate.getPublicKey().matches( - self.client_certificate.getPublicKey() - ) + async def _check_invalid_netloc(self, url: str) -> None: + from scrapy.core.http2.stream import InvalidHostname - assert isinstance(response.ip_address, IPv4Address) - assert str(response.ip_address) == "127.0.0.1" - - d = self.make_request(request) - d.addCallback(assert_metadata) - d.addErrback(self.fail) - - return d - - def _check_invalid_netloc(self, url): request = Request(url) + with pytest.raises(InvalidHostname) as exc_info: + await self.make_request(request) + error_msg = str(exc_info.value) + assert "localhost" in error_msg + assert "127.0.0.1" in error_msg + assert str(request) in error_msg - def assert_invalid_hostname(failure: Failure): - from scrapy.core.http2.stream import InvalidHostname + @deferred_f_from_coro_f + async def test_invalid_hostname(self): + await self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - assert failure.check(InvalidHostname) is not None - error_msg = str(failure.value) - assert "localhost" in error_msg - assert "127.0.0.1" in error_msg - assert str(request) in error_msg - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_invalid_hostname) - return d - - def test_invalid_hostname(self): - return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - - def test_invalid_host_port(self): + @deferred_f_from_coro_f + async def test_invalid_host_port(self): port = self.port_number + 1 - return self._check_invalid_netloc(f"https://127.0.0.1:{port}") + await self._check_invalid_netloc(f"https://127.0.0.1:{port}") - def test_connection_stays_with_invalid_requests(self): - d_list = [ - self.test_invalid_hostname(), - self.test_invalid_host_port(), - self.test_GET_small_body(), - self.test_POST_small_json(), - ] - - return DeferredList(d_list, fireOnOneErrback=True) + @deferred_f_from_coro_f + async def test_connection_stays_with_invalid_requests(self): + await maybe_deferred_to_future(self.test_invalid_hostname()) + await maybe_deferred_to_future(self.test_invalid_host_port()) + await maybe_deferred_to_future(self.test_GET_small_body()) + await maybe_deferred_to_future(self.test_POST_small_json()) + @inlineCallbacks def test_connection_timeout(self): request = Request(self.get_url("/timeout")) - d = self.make_request(request) # Update the timer to 1s to test connection timeout self.client.setTimeout(1) - def assert_timeout_error(failure: Failure): - for err in failure.value.reasons: - from scrapy.core.http2.protocol import H2ClientProtocol + with pytest.raises(ResponseFailed) as exc_info: + yield self.make_request_dfd(request) - if isinstance(err, TimeoutError): - assert ( - f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" - in str(err) - ) - break - else: - pytest.fail("No TimeoutError raised.") + for err in exc_info.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol - d.addCallback(self.fail) - d.addErrback(assert_timeout_error) - return d + if isinstance(err, TimeoutError): + assert ( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" + in str(err) + ) + break + else: + pytest.fail("No TimeoutError raised.") - def test_request_headers_received(self): + @deferred_f_from_coro_f + async def test_request_headers_received(self): request = Request( self.get_url("/request-headers"), headers={"header-1": "header value 1", "header-2": "header value 2"}, ) - d = self.make_request(request) + response = await self.make_request(request) + assert response.status == 200 + assert response.request == request - def assert_request_headers(response: Response): - assert response.status == 200 - assert response.request == request - - response_headers = json.loads(str(response.body, "utf-8")) - assert isinstance(response_headers, dict) - for k, v in request.headers.items(): - k, v = str(k, "utf-8"), str(v[0], "utf-8") - assert k in response_headers - assert v == response_headers[k] - - d.addErrback(self.fail) - d.addCallback(assert_request_headers) - return d + response_headers = json.loads(str(response.body, "utf-8")) + assert isinstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, "utf-8"), str(v[0], "utf-8") + assert k in response_headers + assert v == response_headers[k] diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 00e534c4b..cf827e481 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -5,6 +5,7 @@ from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING, Any +import pytest from testfixtures import LogCapture from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase @@ -218,18 +219,20 @@ class TestFileDownloadCrawl(TestCase): assert "ZeroDivisionError" in str(log) -skip_pillow: str | None +pillow_available: bool try: from PIL import Image # noqa: F401 except ImportError: - skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" + pillow_available = False else: - skip_pillow = None + pillow_available = True -class ImageDownloadCrawlTestCase(TestFileDownloadCrawl): - skip = skip_pillow - +@pytest.mark.skipif( + not pillow_available, + reason="Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", +) +class TestImageDownloadCrawl(TestFileDownloadCrawl): pipeline_class = "scrapy.pipelines.images.ImagesPipeline" store_setting_key = "IMAGES_STORE" media_key = "images" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 492409d02..b4eae108f 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -3,6 +3,7 @@ import os import random import time import warnings +from abc import ABC, abstractmethod from datetime import datetime from io import BytesIO from pathlib import Path @@ -265,7 +266,12 @@ class TestFilesPipeline(unittest.TestCase): assert file_path(request, item=item) == "full/path-to-store-file" -class FilesPipelineTestCaseFieldsMixin: +class TestFilesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) @@ -302,7 +308,7 @@ class FilesPipelineTestCaseFieldsMixin: assert isinstance(item, self.item_class) -class TestFilesPipelineFieldsDict(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsDict(TestFilesPipelineFieldsMixin): item_class = dict @@ -316,7 +322,7 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class TestFilesPipelineFieldsItem(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestItem @@ -331,7 +337,7 @@ class FilesPipelineTestDataClass: custom_files: list = dataclasses.field(default_factory=list) -class TestFilesPipelineFieldsDataClass(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestDataClass @@ -346,7 +352,7 @@ class FilesPipelineTestAttrsItem: custom_files: list[dict[str, str]] = attr.ib(default=list) -class TestFilesPipelineFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsAttrsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestAttrsItem diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f2ee18bd9..74b4495ad 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,8 +3,10 @@ from __future__ import annotations import dataclasses import io import random +from abc import ABC, abstractmethod from shutil import rmtree from tempfile import mkdtemp +from typing import Any import attr import pytest @@ -208,7 +210,12 @@ class TestImagesPipeline: assert converted.getcolors() == [(10000, (205, 230, 255))] -class ImagesPipelineTestCaseFieldsMixin: +class TestImagesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError + def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) @@ -245,7 +252,7 @@ class ImagesPipelineTestCaseFieldsMixin: assert isinstance(item, self.item_class) -class TestImagesPipelineFieldsDict(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsDict(TestImagesPipelineFieldsMixin): item_class = dict @@ -259,7 +266,7 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class TestImagesPipelineFieldsItem(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestItem @@ -274,7 +281,7 @@ class ImagesPipelineTestDataClass: custom_images: list = dataclasses.field(default_factory=list) -class TestImagesPipelineFieldsDataClass(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestDataClass @@ -289,7 +296,7 @@ class ImagesPipelineTestAttrsItem: custom_images: list[dict[str, str]] = attr.ib(default=list) -class TestImagesPipelineFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsAttrsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestAttrsItem diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 2d0db6e25..40149f184 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -345,7 +345,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + req = Request("http://url", meta={"result": "ITSME"}) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) assert new_item["results"] == [(True, "ITSME")] diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5c2772c30..26482fc8d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -115,7 +115,7 @@ class TestMinimalScheduler(InterfaceCheckMixin): assert not self.scheduler.has_pending_requests() -class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): +class TestSimpleScheduler(TestCase, InterfaceCheckMixin): def setUp(self): self.scheduler = SimpleScheduler() @@ -145,7 +145,7 @@ class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): assert close_result == "close" -class MinimalSchedulerCrawlTest(TestCase): +class TestMinimalSchedulerCrawl(TestCase): scheduler_cls = MinimalScheduler @inlineCallbacks @@ -162,5 +162,5 @@ class MinimalSchedulerCrawlTest(TestCase): assert f"'item_scraped_count': {len(PATHS)}" in str(log) -class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): +class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl): scheduler_cls = SimpleScheduler diff --git a/tests/test_signals.py b/tests/test_signals.py index 5a536896e..b20a949e8 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -21,7 +21,7 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class MainTestCase(TestCase): +class TestMain(TestCase): @deferred_f_from_coro_f async def test_scheduler_empty(self): crawler = get_crawler() @@ -35,7 +35,7 @@ class MainTestCase(TestCase): assert len(calls) >= 1 -class MockServerTestCase(TestCase): +class TestMockServer(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 1815aad76..3c7fc65d5 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -18,7 +18,7 @@ ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} -class MainTestCase(TestCase): +class TestMain(TestCase): async def _test_spider(self, spider, expected_items=None): actual_items = [] expected_items = [] if expected_items is None else expected_items diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 725833a49..e1c8b5fec 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -47,7 +47,7 @@ class UniversalSpiderMiddleware: raise NotImplementedError -# Spiders and spider middlewares for MainTestCase._test_wrap +# Spiders and spider middlewares for TestMain._test_wrap class ModernWrapSpider(Spider): @@ -106,7 +106,7 @@ class DeprecatedWrapSpiderMiddleware: yield ITEM_C -class MainTestCase(TestCase): +class TestMain(TestCase): async def _test(self, spider_middlewares, spider_cls, expected_items): actual_items = [] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 372213319..1cfaf70fa 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -92,10 +92,10 @@ class TestDeferUtils(unittest.TestCase): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] + @inlineCallbacks def test_process_parallel_failure(self): - d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") - self.failUnlessFailure(d, TypeError) - return d + with pytest.raises(TypeError): + yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") class TestIterErrback: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 97dade26e..79bac8bc5 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -59,12 +59,12 @@ class TestSendCatchLog(unittest.TestCase): return "OK" -class SendCatchLogDeferredTest(TestSendCatchLog): +class TestSendCatchLogDeferred(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) -class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): +class TestSendCatchLogDeferred2(TestSendCatchLogDeferred): def ok_handler(self, arg, handlers_called): from twisted.internet import reactor @@ -76,7 +76,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): @pytest.mark.usefixtures("reactor_pytest") -class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): +class TestSendCatchLogDeferredAsyncDef(TestSendCatchLogDeferred): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -85,7 +85,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): @pytest.mark.only_asyncio -class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): +class TestSendCatchLogDeferredAsyncio(TestSendCatchLogDeferred): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -93,12 +93,12 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") -class SendCatchLogAsyncTest(TestSendCatchLog): +class TestSendCatchLogAsync(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return deferred_from_coro(send_catch_log_async(signal, *a, **kw)) -class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): +class TestSendCatchLogAsync2(TestSendCatchLogAsync): def ok_handler(self, arg, handlers_called): from twisted.internet import reactor @@ -110,7 +110,7 @@ class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): @pytest.mark.usefixtures("reactor_pytest") -class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): +class TestSendCatchLogAsyncAsyncDef(TestSendCatchLogAsync): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -119,7 +119,7 @@ class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): @pytest.mark.only_asyncio -class SendCatchLogAsyncAsyncioTest(SendCatchLogAsyncTest): +class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 8b32e40bb..569f4f639 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -234,35 +234,31 @@ class TestWebClient(unittest.TestCase): def getURL(self, path): return f"http://127.0.0.1:{self.portno}/{path}" + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s) - ) + body = yield getPage(self.getURL("payload"), body=s) + assert body == to_bytes(s) + @inlineCallbacks def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url - return defer.gatherResults( - [ - getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}") - ), - getPage( - self.getURL("host"), headers={"Host": "www.example.com"} - ).addCallback(self.assertEqual, to_bytes("www.example.com")), - ] - ) + body = yield getPage(self.getURL("host")) + assert body == to_bytes(f"127.0.0.1:{self.portno}") + body = yield getPage(self.getURL("host"), headers={"Host": "www.example.com"}) + assert body == to_bytes("www.example.com") + @inlineCallbacks def test_getPage(self): """ L{client.getPage} returns a L{Deferred} which is called back with the body of the response if the default method B{GET} is used. """ - d = getPage(self.getURL("file")) - d.addCallback(self.assertEqual, b"0123456789") - return d + body = yield getPage(self.getURL("file")) + assert body == b"0123456789" + @inlineCallbacks def test_getPageHead(self): """ L{client.getPage} returns a L{Deferred} which is called back with @@ -273,22 +269,20 @@ class TestWebClient(unittest.TestCase): def _getPage(method): return getPage(self.getURL("file"), method=method) - return defer.gatherResults( - [ - _getPage("head").addCallback(self.assertEqual, b""), - _getPage("HEAD").addCallback(self.assertEqual, b""), - ] - ) + body = yield _getPage("head") + assert body == b"" + body = yield _getPage("HEAD") + assert body == b"" + @inlineCallbacks def test_timeoutNotTriggering(self): """ When a non-zero timeout is passed to L{getPage} and the page is retrieved before the timeout period elapses, the L{Deferred} is called back with the contents of the page. """ - d = getPage(self.getURL("host"), timeout=100) - d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) - return d + body = yield getPage(self.getURL("host"), timeout=100) + assert body == to_bytes(f"127.0.0.1:{self.portno}") @inlineCallbacks def test_timeoutTriggering(self): @@ -307,12 +301,12 @@ class TestWebClient(unittest.TestCase): if connected: connected[0].transport.loseConnection() + @inlineCallbacks def testNotFound(self): - return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile) - - def _cbNoSuchFile(self, pageData): - assert b"404 - No Such Resource" in pageData + body = yield getPage(self.getURL("notsuchfile")) + assert b"404 - No Such Resource" in body + @inlineCallbacks def testFactoryInfo(self): from twisted.internet import reactor @@ -320,63 +314,60 @@ class TestWebClient(unittest.TestCase): parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(parsed.hostname, parsed.port, factory) - return factory.deferred.addCallback(self._cbFactoryInfo, factory) - - def _cbFactoryInfo(self, ignoredResult, factory): + yield factory.deferred assert factory.status == b"200" assert factory.version.startswith(b"HTTP/") assert factory.message == b"OK" assert factory.response_headers[b"content-length"] == b"10" + @inlineCallbacks def testRedirect(self): - return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) - - def _cbRedirect(self, pageData): + body = yield getPage(self.getURL("redirect")) assert ( - pageData + body == b'\n\n \n \n' b' \n \n ' b'click here\n \n\n' ) + @inlineCallbacks def test_encoding(self): """Test that non-standart body encoding matches Content-Encoding header""" - body = b"\xd0\x81\xd1\x8e\xd0\xaf" - dfd = getPage( - self.getURL("encoding"), body=body, response_transform=lambda r: r + original_body = b"\xd0\x81\xd1\x8e\xd0\xaf" + response = yield getPage( + self.getURL("encoding"), body=original_body, response_transform=lambda r: r ) - return dfd.addCallback(self._check_Encoding, body) - - def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b"Content-Encoding"]) assert content_encoding == EncodingResource.out_encoding assert response.body.decode(content_encoding) == to_unicode(original_body) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientSSLTestCase(TestContextFactoryBase): +class TestWebClientSSL(TestContextFactoryBase): + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s) - ) + body = yield getPage(self.getURL("payload"), body=s) + assert body == to_bytes(s) -class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): +class TestWebClientCustomCiphersSSL(TestWebClientSSL): # we try to use a cipher that is not enabled by default in OpenSSL custom_ciphers = "CAMELLIA256-SHA" context_factory = ssl_context_factory(cipher_string=custom_ciphers) + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} ) client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - return getPage( + body = yield getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory - ).addCallback(self.assertEqual, to_bytes(s)) + ) + assert body == to_bytes(s) @inlineCallbacks def testPayloadDisabledCipher(self):