mirror of https://github.com/scrapy/scrapy.git
Further reduce deps on unittest. (#6884)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
ac956f8595
commit
b4d11b8b25
|
|
@ -39,23 +39,21 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase):
|
|||
pass
|
||||
|
||||
|
||||
class Https11WrongHostnameTestCase(
|
||||
HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase
|
||||
):
|
||||
class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
pass
|
||||
|
||||
|
||||
class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
pass
|
||||
|
||||
|
||||
class Https11InvalidDNSPattern(
|
||||
class TestHttps11InvalidDNSPattern(
|
||||
HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -163,23 +163,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
assert json.loads(response.text)["headers"][header] == [value1, value2]
|
||||
|
||||
|
||||
class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
pass
|
||||
|
||||
|
||||
class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
pass
|
||||
|
||||
|
||||
class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase):
|
||||
class TestHttps2InvalidDNSPattern(
|
||||
H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
pass
|
||||
|
||||
|
||||
class Http2MockServerTestCase(TestHttpMockServerBase):
|
||||
class TestHttp2MockServer(TestHttpMockServerBase):
|
||||
"""HTTP 2.0 test case with MockServer"""
|
||||
|
||||
@property
|
||||
|
|
@ -193,7 +195,7 @@ class Http2MockServerTestCase(TestHttpMockServerBase):
|
|||
is_secure = True
|
||||
|
||||
|
||||
class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase):
|
||||
class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
|
||||
# only used for HTTPS tests
|
||||
keyfile = "keys/localhost.key"
|
||||
certfile = "keys/localhost.crt"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from unittest import mock
|
|||
|
||||
import pytest
|
||||
from twisted.cred import checkers, credentials, portal
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.protocols.ftp import FTPFactory, FTPRealm
|
||||
from twisted.trial import unittest
|
||||
from w3lib.url import path_to_file_uri
|
||||
|
|
@ -340,9 +341,10 @@ class TestFTPBase(unittest.TestCase):
|
|||
self.portNum = self.port.getHost().port
|
||||
crawler = get_crawler()
|
||||
self.download_handler = build_from_crawler(FTPDownloadHandler, crawler)
|
||||
self.addCleanup(self.port.stopListening)
|
||||
|
||||
@inlineCallbacks
|
||||
def tearDown(self):
|
||||
yield self.port.stopListening()
|
||||
shutil.rmtree(self.directory)
|
||||
|
||||
def _add_test_callbacks(self, deferred, callback=None, errback=None):
|
||||
|
|
@ -478,9 +480,10 @@ class TestAnonymousFTP(TestFTPBase):
|
|||
self.portNum = self.port.getHost().port
|
||||
crawler = get_crawler()
|
||||
self.download_handler = build_from_crawler(FTPDownloadHandler, crawler)
|
||||
self.addCleanup(self.port.stopListening)
|
||||
|
||||
@inlineCallbacks
|
||||
def tearDown(self):
|
||||
yield self.port.stopListening()
|
||||
shutil.rmtree(self.directory)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -247,10 +247,8 @@ Disallow: /some/randome/page.html
|
|||
assert request.callback == NO_CALLBACK
|
||||
|
||||
|
||||
@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
|
||||
class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware):
|
||||
if not rerp_available():
|
||||
skip = "Rerp parser is not installed"
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.crawler.settings.set(
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
|
|||
self.times[slot].append(time.time())
|
||||
|
||||
|
||||
class CrawlTestCase(TestCase):
|
||||
class TestCrawl(TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ async def sleep(seconds: float = 0.001) -> None:
|
|||
await maybe_deferred_to_future(deferred)
|
||||
|
||||
|
||||
class MainTestCase(TestCase):
|
||||
class TestMain(TestCase):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_sleep(self):
|
||||
"""Neither asynchronous sleeps on Spider.start() nor the equivalent on
|
||||
|
|
@ -119,7 +119,7 @@ class MainTestCase(TestCase):
|
|||
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
|
||||
|
||||
|
||||
class RequestSendOrderTestCase(TestCase):
|
||||
class TestRequestSendOrder(TestCase):
|
||||
seconds = 0.1 # increase if flaky
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from scrapy.extensions.telnet import TelnetConsole
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class TelnetExtensionTest(unittest.TestCase):
|
||||
class TestTelnetExtension(unittest.TestCase):
|
||||
def _get_console_and_portal(self, settings=None):
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
console = TelnetConsole(crawler)
|
||||
|
|
|
|||
|
|
@ -91,52 +91,53 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
|
|||
return (client_mock, bucket_mock, blob_mock)
|
||||
|
||||
|
||||
# TODO: replace self.mktemp() and drop the unittest.TestCase base
|
||||
class TestFileFeedStorage(unittest.TestCase):
|
||||
def test_store_file_uri(self):
|
||||
path = Path(self.mktemp()).resolve()
|
||||
class TestFileFeedStorage:
|
||||
def test_store_file_uri(self, tmp_path):
|
||||
path = tmp_path / "file.txt"
|
||||
uri = path_to_file_uri(str(path))
|
||||
self._assert_stores(FileFeedStorage(uri), path)
|
||||
|
||||
def test_store_file_uri_makedirs(self):
|
||||
path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt"
|
||||
def test_store_file_uri_makedirs(self, tmp_path):
|
||||
path = tmp_path / "more" / "paths" / "file.txt"
|
||||
uri = path_to_file_uri(str(path))
|
||||
self._assert_stores(FileFeedStorage(uri), path)
|
||||
|
||||
def test_store_direct_path(self):
|
||||
path = Path(self.mktemp()).resolve()
|
||||
def test_store_direct_path(self, tmp_path):
|
||||
path = tmp_path / "file.txt"
|
||||
self._assert_stores(FileFeedStorage(str(path)), path)
|
||||
|
||||
def test_store_direct_path_relative(self):
|
||||
path = Path(self.mktemp())
|
||||
def test_store_direct_path_relative(self, tmp_path):
|
||||
path = (tmp_path / "foo" / "bar").relative_to(Path.cwd())
|
||||
self._assert_stores(FileFeedStorage(str(path)), path)
|
||||
|
||||
def test_interface(self):
|
||||
path = self.mktemp()
|
||||
st = FileFeedStorage(path)
|
||||
def test_interface(self, tmp_path):
|
||||
path = tmp_path / "file.txt"
|
||||
st = FileFeedStorage(str(path))
|
||||
verifyObject(IFeedStorage, st)
|
||||
|
||||
def _store(self, feed_options=None) -> Path:
|
||||
path = Path(self.mktemp()).resolve()
|
||||
@staticmethod
|
||||
def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:
|
||||
storage = FileFeedStorage(str(path), feed_options=feed_options)
|
||||
spider = scrapy.Spider("default")
|
||||
file = storage.open(spider)
|
||||
file.write(b"content")
|
||||
storage.store(file)
|
||||
return path
|
||||
|
||||
def test_append(self):
|
||||
path = self._store()
|
||||
def test_append(self, tmp_path):
|
||||
path = tmp_path / "file.txt"
|
||||
self._store(path)
|
||||
self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent")
|
||||
|
||||
def test_overwrite(self):
|
||||
path = self._store({"overwrite": True})
|
||||
def test_overwrite(self, tmp_path):
|
||||
path = tmp_path / "file.txt"
|
||||
self._store(path, {"overwrite": True})
|
||||
self._assert_stores(
|
||||
FileFeedStorage(str(path), feed_options={"overwrite": True}), path
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _assert_stores(
|
||||
self, storage: FileFeedStorage, path: Path, expected_content: bytes = b"content"
|
||||
storage: FileFeedStorage, path: Path, expected_content: bytes = b"content"
|
||||
) -> None:
|
||||
spider = scrapy.Spider("default")
|
||||
file = storage.open(spider)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import string
|
|||
from ipaddress import IPv4Address
|
||||
from pathlib import Path
|
||||
from tempfile import mkdtemp
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
from unittest import mock
|
||||
from urllib.parse import urlencode
|
||||
|
||||
|
|
@ -32,17 +32,22 @@ from twisted.web.static import File
|
|||
from scrapy.http import JsonRequest, Request, Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.defer import (
|
||||
deferred_f_from_coro_f,
|
||||
deferred_from_coro,
|
||||
maybe_deferred_to_future,
|
||||
)
|
||||
from tests.mockserver import LeafResource, Status, ssl_context_factory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from twisted.python.failure import Failure
|
||||
from collections.abc import Coroutine
|
||||
|
||||
|
||||
def generate_random_string(size):
|
||||
def generate_random_string(size: int) -> str:
|
||||
return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))
|
||||
|
||||
|
||||
def make_html_body(val):
|
||||
def make_html_body(val: str) -> bytes:
|
||||
response = f"""<html>
|
||||
<h1>Hello from HTTP2<h1>
|
||||
<p>{val}</p>
|
||||
|
|
@ -92,7 +97,7 @@ class GetDataHtmlLarge(LeafResource):
|
|||
|
||||
class PostDataJsonMixin:
|
||||
@staticmethod
|
||||
def make_response(request: TxRequest, extra_data: str):
|
||||
def make_response(request: TxRequest, extra_data: str) -> bytes:
|
||||
assert request.content is not None
|
||||
response = {
|
||||
"request-headers": {},
|
||||
|
|
@ -179,7 +184,6 @@ def get_client_certificate(
|
|||
pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
return PrivateCertificate.loadPEM(pem)
|
||||
|
||||
|
||||
|
|
@ -238,6 +242,7 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8"))
|
||||
|
||||
self.conn_closed_deferred = Deferred()
|
||||
|
||||
from scrapy.core.http2.protocol import H2ClientFactory
|
||||
|
||||
h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
|
||||
|
|
@ -255,7 +260,7 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
shutil.rmtree(self.temp_directory)
|
||||
self.conn_closed_deferred = None
|
||||
|
||||
def get_url(self, path):
|
||||
def get_url(self, path: str) -> str:
|
||||
"""
|
||||
:param path: Should have / at the starting compulsorily if not empty
|
||||
:return: Complete url
|
||||
|
|
@ -264,143 +269,146 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
assert path[0] == "/" or path[0] == "&"
|
||||
return f"{self.scheme}://{self.hostname}:{self.port_number}{path}"
|
||||
|
||||
def make_request(self, request: Request) -> Deferred:
|
||||
async def make_request(self, request: Request) -> Response:
|
||||
return await maybe_deferred_to_future(self.make_request_dfd(request))
|
||||
|
||||
def make_request_dfd(self, request: Request) -> Deferred[Response]:
|
||||
return self.client.request(request, DummySpider())
|
||||
|
||||
@staticmethod
|
||||
def _check_repeat(get_deferred, count):
|
||||
async def _check_repeat(
|
||||
get_coro: Callable[[], Coroutine[Any, Any, None]], count: int
|
||||
) -> None:
|
||||
d_list = []
|
||||
for _ in range(count):
|
||||
d = get_deferred()
|
||||
d = deferred_from_coro(get_coro())
|
||||
d_list.append(d)
|
||||
|
||||
return DeferredList(d_list, fireOnOneErrback=True)
|
||||
await maybe_deferred_to_future(DeferredList(d_list, fireOnOneErrback=True))
|
||||
|
||||
def _check_GET(self, request: Request, expected_body, expected_status):
|
||||
def check_response(response: Response):
|
||||
assert response.status == expected_status
|
||||
assert response.body == expected_body
|
||||
assert response.request == request
|
||||
async def _check_GET(
|
||||
self, request: Request, expected_body: bytes, expected_status: int
|
||||
) -> None:
|
||||
response = await self.make_request(request)
|
||||
assert response.status == expected_status
|
||||
assert response.body == expected_body
|
||||
assert response.request == request
|
||||
|
||||
content_length_header = response.headers.get("Content-Length")
|
||||
assert content_length_header is not None
|
||||
content_length = int(content_length_header)
|
||||
assert len(response.body) == content_length
|
||||
content_length_header = response.headers.get("Content-Length")
|
||||
assert content_length_header is not None
|
||||
content_length = int(content_length_header)
|
||||
assert len(response.body) == content_length
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(check_response)
|
||||
d.addErrback(self.fail)
|
||||
return d
|
||||
|
||||
def test_GET_small_body(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_GET_small_body(self):
|
||||
request = Request(self.get_url("/get-data-html-small"))
|
||||
return self._check_GET(request, Data.HTML_SMALL, 200)
|
||||
await self._check_GET(request, Data.HTML_SMALL, 200)
|
||||
|
||||
def test_GET_large_body(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_GET_large_body(self):
|
||||
request = Request(self.get_url("/get-data-html-large"))
|
||||
return self._check_GET(request, Data.HTML_LARGE, 200)
|
||||
await self._check_GET(request, Data.HTML_LARGE, 200)
|
||||
|
||||
def _check_GET_x10(self, *args, **kwargs):
|
||||
def get_deferred():
|
||||
return self._check_GET(*args, **kwargs)
|
||||
async def _check_GET_x10(
|
||||
self, request: Request, expected_body: bytes, expected_status: int
|
||||
) -> None:
|
||||
async def get_coro() -> None:
|
||||
await self._check_GET(request, expected_body, expected_status)
|
||||
|
||||
return self._check_repeat(get_deferred, 10)
|
||||
await self._check_repeat(get_coro, 10)
|
||||
|
||||
def test_GET_small_body_x10(self):
|
||||
return self._check_GET_x10(
|
||||
@deferred_f_from_coro_f
|
||||
async def test_GET_small_body_x10(self):
|
||||
await self._check_GET_x10(
|
||||
Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200
|
||||
)
|
||||
|
||||
def test_GET_large_body_x10(self):
|
||||
return self._check_GET_x10(
|
||||
@deferred_f_from_coro_f
|
||||
async def test_GET_large_body_x10(self):
|
||||
await self._check_GET_x10(
|
||||
Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200
|
||||
)
|
||||
|
||||
def _check_POST_json(
|
||||
async def _check_POST_json(
|
||||
self,
|
||||
request: Request,
|
||||
expected_request_body,
|
||||
expected_extra_data,
|
||||
expected_request_body: dict[str, str],
|
||||
expected_extra_data: str,
|
||||
expected_status: int,
|
||||
):
|
||||
d = self.make_request(request)
|
||||
) -> None:
|
||||
response = await self.make_request(request)
|
||||
|
||||
def assert_response(response: Response):
|
||||
assert response.status == expected_status
|
||||
assert response.request == request
|
||||
assert response.status == expected_status
|
||||
assert response.request == request
|
||||
|
||||
content_length_header = response.headers.get("Content-Length")
|
||||
assert content_length_header is not None
|
||||
content_length = int(content_length_header)
|
||||
assert len(response.body) == content_length
|
||||
content_length_header = response.headers.get("Content-Length")
|
||||
assert content_length_header is not None
|
||||
content_length = int(content_length_header)
|
||||
assert len(response.body) == content_length
|
||||
|
||||
# Parse the body
|
||||
content_encoding_header = response.headers[b"Content-Encoding"]
|
||||
assert content_encoding_header is not None
|
||||
content_encoding = str(content_encoding_header, "utf-8")
|
||||
body = json.loads(str(response.body, content_encoding))
|
||||
assert "request-body" in body
|
||||
assert "extra-data" in body
|
||||
assert "request-headers" in body
|
||||
# Parse the body
|
||||
content_encoding_header = response.headers[b"Content-Encoding"]
|
||||
assert content_encoding_header is not None
|
||||
content_encoding = str(content_encoding_header, "utf-8")
|
||||
body = json.loads(str(response.body, content_encoding))
|
||||
assert "request-body" in body
|
||||
assert "extra-data" in body
|
||||
assert "request-headers" in body
|
||||
|
||||
request_body = body["request-body"]
|
||||
assert request_body == expected_request_body
|
||||
request_body = body["request-body"]
|
||||
assert request_body == expected_request_body
|
||||
|
||||
extra_data = body["extra-data"]
|
||||
assert extra_data == expected_extra_data
|
||||
extra_data = body["extra-data"]
|
||||
assert extra_data == expected_extra_data
|
||||
|
||||
# Check if headers were sent successfully
|
||||
request_headers = body["request-headers"]
|
||||
for k, v in request.headers.items():
|
||||
k_str = str(k, "utf-8")
|
||||
assert k_str in request_headers
|
||||
assert request_headers[k_str] == str(v[0], "utf-8")
|
||||
# Check if headers were sent successfully
|
||||
request_headers = body["request-headers"]
|
||||
for k, v in request.headers.items():
|
||||
k_str = str(k, "utf-8")
|
||||
assert k_str in request_headers
|
||||
assert request_headers[k_str] == str(v[0], "utf-8")
|
||||
|
||||
d.addCallback(assert_response)
|
||||
d.addErrback(self.fail)
|
||||
return d
|
||||
|
||||
def test_POST_small_json(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_POST_small_json(self):
|
||||
request = JsonRequest(
|
||||
url=self.get_url("/post-data-json-small"),
|
||||
method="POST",
|
||||
data=Data.JSON_SMALL,
|
||||
)
|
||||
return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200)
|
||||
await self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200)
|
||||
|
||||
def test_POST_large_json(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_POST_large_json(self):
|
||||
request = JsonRequest(
|
||||
url=self.get_url("/post-data-json-large"),
|
||||
method="POST",
|
||||
data=Data.JSON_LARGE,
|
||||
)
|
||||
return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200)
|
||||
await self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200)
|
||||
|
||||
def _check_POST_json_x10(self, *args, **kwargs):
|
||||
def get_deferred():
|
||||
return self._check_POST_json(*args, **kwargs)
|
||||
async def _check_POST_json_x10(self, *args, **kwargs):
|
||||
async def get_coro() -> None:
|
||||
await self._check_POST_json(*args, **kwargs)
|
||||
|
||||
return self._check_repeat(get_deferred, 10)
|
||||
await self._check_repeat(get_coro, 10)
|
||||
|
||||
def test_POST_small_json_x10(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_POST_small_json_x10(self):
|
||||
request = JsonRequest(
|
||||
url=self.get_url("/post-data-json-small"),
|
||||
method="POST",
|
||||
data=Data.JSON_SMALL,
|
||||
)
|
||||
return self._check_POST_json_x10(
|
||||
request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200
|
||||
)
|
||||
await self._check_POST_json_x10(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200)
|
||||
|
||||
def test_POST_large_json_x10(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_POST_large_json_x10(self):
|
||||
request = JsonRequest(
|
||||
url=self.get_url("/post-data-json-large"),
|
||||
method="POST",
|
||||
data=Data.JSON_LARGE,
|
||||
)
|
||||
return self._check_POST_json_x10(
|
||||
request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200
|
||||
)
|
||||
await self._check_POST_json_x10(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_invalid_negotiated_protocol(self):
|
||||
|
|
@ -409,77 +417,59 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
):
|
||||
request = Request(url=self.get_url("/status?n=200"))
|
||||
with pytest.raises(ResponseFailed):
|
||||
yield self.make_request(request)
|
||||
yield self.make_request_dfd(request)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_cancel_request(self):
|
||||
request = Request(url=self.get_url("/get-data-html-large"))
|
||||
|
||||
def assert_response(response: Response):
|
||||
assert response.status == 499
|
||||
assert response.request == request
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(assert_response)
|
||||
d.addErrback(self.fail)
|
||||
d = self.make_request_dfd(request)
|
||||
d.cancel()
|
||||
response = yield d
|
||||
assert response.status == 499
|
||||
assert response.request == request
|
||||
|
||||
return d
|
||||
|
||||
def test_download_maxsize_exceeded(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_download_maxsize_exceeded(self):
|
||||
request = Request(
|
||||
url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000}
|
||||
)
|
||||
with pytest.raises(CancelledError) as exc_info:
|
||||
await self.make_request(request)
|
||||
error_pattern = re.compile(
|
||||
rf"Cancelling download of {request.url}: received response "
|
||||
rf"size \(\d*\) larger than download max size \(1000\)"
|
||||
)
|
||||
assert len(re.findall(error_pattern, str(exc_info.value))) == 1
|
||||
|
||||
def assert_cancelled_error(failure):
|
||||
assert isinstance(failure.value, CancelledError)
|
||||
error_pattern = re.compile(
|
||||
rf"Cancelling download of {request.url}: received response "
|
||||
rf"size \(\d*\) larger than download max size \(1000\)"
|
||||
)
|
||||
assert len(re.findall(error_pattern, str(failure.value))) == 1
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(self.fail)
|
||||
d.addErrback(assert_cancelled_error)
|
||||
return d
|
||||
|
||||
@inlineCallbacks
|
||||
def test_received_dataloss_response(self):
|
||||
"""In case when value of Header Content-Length != len(Received Data)
|
||||
ProtocolError is raised"""
|
||||
from h2.exceptions import InvalidBodyLengthError
|
||||
|
||||
request = Request(url=self.get_url("/dataloss"))
|
||||
with pytest.raises(ResponseFailed) as exc_info:
|
||||
yield self.make_request_dfd(request)
|
||||
assert len(exc_info.value.reasons) > 0
|
||||
assert any(
|
||||
isinstance(error, InvalidBodyLengthError)
|
||||
for error in exc_info.value.reasons
|
||||
)
|
||||
|
||||
def assert_failure(failure: Failure):
|
||||
assert len(failure.value.reasons) > 0
|
||||
from h2.exceptions import InvalidBodyLengthError
|
||||
|
||||
assert any(
|
||||
isinstance(error, InvalidBodyLengthError)
|
||||
for error in failure.value.reasons
|
||||
)
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(self.fail)
|
||||
d.addErrback(assert_failure)
|
||||
return d
|
||||
|
||||
def test_missing_content_length_header(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_missing_content_length_header(self):
|
||||
request = Request(url=self.get_url("/no-content-length-header"))
|
||||
response = await self.make_request(request)
|
||||
assert response.status == 200
|
||||
assert response.body == Data.NO_CONTENT_LENGTH
|
||||
assert response.request == request
|
||||
assert "Content-Length" not in response.headers
|
||||
|
||||
def assert_content_length(response: Response):
|
||||
assert response.status == 200
|
||||
assert response.body == Data.NO_CONTENT_LENGTH
|
||||
assert response.request == request
|
||||
assert "Content-Length" not in response.headers
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(assert_content_length)
|
||||
d.addErrback(self.fail)
|
||||
return d
|
||||
|
||||
@inlineCallbacks
|
||||
def _check_log_warnsize(self, request, warn_pattern, expected_body):
|
||||
async def _check_log_warnsize(
|
||||
self, request: Request, warn_pattern: re.Pattern[str], expected_body: bytes
|
||||
) -> None:
|
||||
with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm:
|
||||
response = yield self.make_request(request)
|
||||
response = await self.make_request(request)
|
||||
assert response.status == 200
|
||||
assert response.request == request
|
||||
assert response.body == expected_body
|
||||
|
|
@ -487,8 +477,8 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
# Check the warning is raised only once for this request
|
||||
assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1
|
||||
|
||||
@inlineCallbacks
|
||||
def test_log_expected_warnsize(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_log_expected_warnsize(self):
|
||||
request = Request(
|
||||
url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000}
|
||||
)
|
||||
|
|
@ -497,10 +487,10 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
rf"download warn size \(1000\) in request {request}"
|
||||
)
|
||||
|
||||
yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)
|
||||
await self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_log_received_warnsize(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_log_received_warnsize(self):
|
||||
request = Request(
|
||||
url=self.get_url("/no-content-length-header"),
|
||||
meta={"download_warnsize": 10},
|
||||
|
|
@ -510,20 +500,22 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
rf"warn size \(10\) in request {request}"
|
||||
)
|
||||
|
||||
yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
|
||||
await self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
|
||||
|
||||
def test_max_concurrent_streams(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_max_concurrent_streams(self):
|
||||
"""Send 500 requests at one to check if we can handle
|
||||
very large number of request.
|
||||
"""
|
||||
|
||||
def get_deferred():
|
||||
return self._check_GET(
|
||||
async def get_coro() -> None:
|
||||
await self._check_GET(
|
||||
Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200
|
||||
)
|
||||
|
||||
return self._check_repeat(get_deferred, 500)
|
||||
await self._check_repeat(get_coro, 500)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_inactive_stream(self):
|
||||
"""Here we send 110 requests considering the MAX_CONCURRENT_STREAMS
|
||||
by default is 100. After sending the first 100 requests we close the
|
||||
|
|
@ -532,6 +524,7 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
|
||||
def assert_inactive_stream(failure):
|
||||
assert failure.check(ResponseFailed) is not None
|
||||
|
||||
from scrapy.core.http2.stream import InactiveStreamClosed
|
||||
|
||||
assert any(
|
||||
|
|
@ -540,14 +533,14 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
|
||||
# Send 100 request (we do not check the result)
|
||||
for _ in range(100):
|
||||
d = self.make_request(Request(self.get_url("/get-data-html-small")))
|
||||
d = self.make_request_dfd(Request(self.get_url("/get-data-html-small")))
|
||||
d.addBoth(lambda _: None)
|
||||
d_list.append(d)
|
||||
|
||||
# Now send 10 extra request and save the response deferred in a list
|
||||
for _ in range(10):
|
||||
d = self.make_request(Request(self.get_url("/get-data-html-small")))
|
||||
d.addCallback(self.fail)
|
||||
d = self.make_request_dfd(Request(self.get_url("/get-data-html-small")))
|
||||
d.addCallback(lambda _: pytest.fail("This request should have failed"))
|
||||
d.addErrback(assert_inactive_stream)
|
||||
d_list.append(d)
|
||||
|
||||
|
|
@ -555,13 +548,15 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
# with InactiveStreamClosed
|
||||
self.client.transport.loseConnection()
|
||||
|
||||
return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
|
||||
yield DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
|
||||
|
||||
def test_invalid_request_type(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_invalid_request_type(self):
|
||||
with pytest.raises(TypeError):
|
||||
self.make_request("https://InvalidDataTypePassed.com")
|
||||
await self.make_request("https://InvalidDataTypePassed.com")
|
||||
|
||||
def test_query_parameters(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_query_parameters(self):
|
||||
params = {
|
||||
"a": generate_random_string(20),
|
||||
"b": generate_random_string(20),
|
||||
|
|
@ -569,133 +564,96 @@ class TestHttps2ClientProtocol(TestCase):
|
|||
"d": generate_random_string(20),
|
||||
}
|
||||
request = Request(self.get_url(f"/query-params?{urlencode(params)}"))
|
||||
response = await self.make_request(request)
|
||||
content_encoding_header = response.headers[b"Content-Encoding"]
|
||||
assert content_encoding_header is not None
|
||||
content_encoding = str(content_encoding_header, "utf-8")
|
||||
data = json.loads(str(response.body, content_encoding))
|
||||
assert data == params
|
||||
|
||||
def assert_query_params(response: Response):
|
||||
content_encoding_header = response.headers[b"Content-Encoding"]
|
||||
assert content_encoding_header is not None
|
||||
content_encoding = str(content_encoding_header, "utf-8")
|
||||
data = json.loads(str(response.body, content_encoding))
|
||||
assert data == params
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(assert_query_params)
|
||||
d.addErrback(self.fail)
|
||||
|
||||
return d
|
||||
|
||||
def test_status_codes(self):
|
||||
def assert_response_status(response: Response, expected_status: int):
|
||||
assert response.status == expected_status
|
||||
|
||||
d_list = []
|
||||
@deferred_f_from_coro_f
|
||||
async def test_status_codes(self):
|
||||
for status in [200, 404]:
|
||||
request = Request(self.get_url(f"/status?n={status}"))
|
||||
d = self.make_request(request)
|
||||
d.addCallback(assert_response_status, status)
|
||||
d.addErrback(self.fail)
|
||||
d_list.append(d)
|
||||
response = await self.make_request(request)
|
||||
assert response.status == status
|
||||
|
||||
return DeferredList(d_list, fireOnOneErrback=True)
|
||||
|
||||
def test_response_has_correct_certificate_ip_address(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_response_has_correct_certificate_ip_address(self):
|
||||
request = Request(self.get_url("/status?n=200"))
|
||||
response = await self.make_request(request)
|
||||
assert response.request == request
|
||||
assert isinstance(response.certificate, Certificate)
|
||||
assert response.certificate.original is not None
|
||||
assert response.certificate.getIssuer() == self.client_certificate.getIssuer()
|
||||
assert response.certificate.getPublicKey().matches(
|
||||
self.client_certificate.getPublicKey()
|
||||
)
|
||||
assert isinstance(response.ip_address, IPv4Address)
|
||||
assert str(response.ip_address) == "127.0.0.1"
|
||||
|
||||
def assert_metadata(response: Response):
|
||||
assert response.request == request
|
||||
assert isinstance(response.certificate, Certificate)
|
||||
assert response.certificate.original is not None
|
||||
assert (
|
||||
response.certificate.getIssuer() == self.client_certificate.getIssuer()
|
||||
)
|
||||
assert response.certificate.getPublicKey().matches(
|
||||
self.client_certificate.getPublicKey()
|
||||
)
|
||||
async def _check_invalid_netloc(self, url: str) -> None:
|
||||
from scrapy.core.http2.stream import InvalidHostname
|
||||
|
||||
assert isinstance(response.ip_address, IPv4Address)
|
||||
assert str(response.ip_address) == "127.0.0.1"
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(assert_metadata)
|
||||
d.addErrback(self.fail)
|
||||
|
||||
return d
|
||||
|
||||
def _check_invalid_netloc(self, url):
|
||||
request = Request(url)
|
||||
with pytest.raises(InvalidHostname) as exc_info:
|
||||
await self.make_request(request)
|
||||
error_msg = str(exc_info.value)
|
||||
assert "localhost" in error_msg
|
||||
assert "127.0.0.1" in error_msg
|
||||
assert str(request) in error_msg
|
||||
|
||||
def assert_invalid_hostname(failure: Failure):
|
||||
from scrapy.core.http2.stream import InvalidHostname
|
||||
@deferred_f_from_coro_f
|
||||
async def test_invalid_hostname(self):
|
||||
await self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain")
|
||||
|
||||
assert failure.check(InvalidHostname) is not None
|
||||
error_msg = str(failure.value)
|
||||
assert "localhost" in error_msg
|
||||
assert "127.0.0.1" in error_msg
|
||||
assert str(request) in error_msg
|
||||
|
||||
d = self.make_request(request)
|
||||
d.addCallback(self.fail)
|
||||
d.addErrback(assert_invalid_hostname)
|
||||
return d
|
||||
|
||||
def test_invalid_hostname(self):
|
||||
return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain")
|
||||
|
||||
def test_invalid_host_port(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_invalid_host_port(self):
|
||||
port = self.port_number + 1
|
||||
return self._check_invalid_netloc(f"https://127.0.0.1:{port}")
|
||||
await self._check_invalid_netloc(f"https://127.0.0.1:{port}")
|
||||
|
||||
def test_connection_stays_with_invalid_requests(self):
|
||||
d_list = [
|
||||
self.test_invalid_hostname(),
|
||||
self.test_invalid_host_port(),
|
||||
self.test_GET_small_body(),
|
||||
self.test_POST_small_json(),
|
||||
]
|
||||
|
||||
return DeferredList(d_list, fireOnOneErrback=True)
|
||||
@deferred_f_from_coro_f
|
||||
async def test_connection_stays_with_invalid_requests(self):
|
||||
await maybe_deferred_to_future(self.test_invalid_hostname())
|
||||
await maybe_deferred_to_future(self.test_invalid_host_port())
|
||||
await maybe_deferred_to_future(self.test_GET_small_body())
|
||||
await maybe_deferred_to_future(self.test_POST_small_json())
|
||||
|
||||
@inlineCallbacks
|
||||
def test_connection_timeout(self):
|
||||
request = Request(self.get_url("/timeout"))
|
||||
d = self.make_request(request)
|
||||
|
||||
# Update the timer to 1s to test connection timeout
|
||||
self.client.setTimeout(1)
|
||||
|
||||
def assert_timeout_error(failure: Failure):
|
||||
for err in failure.value.reasons:
|
||||
from scrapy.core.http2.protocol import H2ClientProtocol
|
||||
with pytest.raises(ResponseFailed) as exc_info:
|
||||
yield self.make_request_dfd(request)
|
||||
|
||||
if isinstance(err, TimeoutError):
|
||||
assert (
|
||||
f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"
|
||||
in str(err)
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("No TimeoutError raised.")
|
||||
for err in exc_info.value.reasons:
|
||||
from scrapy.core.http2.protocol import H2ClientProtocol
|
||||
|
||||
d.addCallback(self.fail)
|
||||
d.addErrback(assert_timeout_error)
|
||||
return d
|
||||
if isinstance(err, TimeoutError):
|
||||
assert (
|
||||
f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"
|
||||
in str(err)
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("No TimeoutError raised.")
|
||||
|
||||
def test_request_headers_received(self):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_request_headers_received(self):
|
||||
request = Request(
|
||||
self.get_url("/request-headers"),
|
||||
headers={"header-1": "header value 1", "header-2": "header value 2"},
|
||||
)
|
||||
d = self.make_request(request)
|
||||
response = await self.make_request(request)
|
||||
assert response.status == 200
|
||||
assert response.request == request
|
||||
|
||||
def assert_request_headers(response: Response):
|
||||
assert response.status == 200
|
||||
assert response.request == request
|
||||
|
||||
response_headers = json.loads(str(response.body, "utf-8"))
|
||||
assert isinstance(response_headers, dict)
|
||||
for k, v in request.headers.items():
|
||||
k, v = str(k, "utf-8"), str(v[0], "utf-8")
|
||||
assert k in response_headers
|
||||
assert v == response_headers[k]
|
||||
|
||||
d.addErrback(self.fail)
|
||||
d.addCallback(assert_request_headers)
|
||||
return d
|
||||
response_headers = json.loads(str(response.body, "utf-8"))
|
||||
assert isinstance(response_headers, dict)
|
||||
for k, v in request.headers.items():
|
||||
k, v = str(k, "utf-8"), str(v[0], "utf-8")
|
||||
assert k in response_headers
|
||||
assert v == response_headers[k]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from pathlib import Path
|
|||
from tempfile import mkdtemp
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
|
@ -218,18 +219,20 @@ class TestFileDownloadCrawl(TestCase):
|
|||
assert "ZeroDivisionError" in str(log)
|
||||
|
||||
|
||||
skip_pillow: str | None
|
||||
pillow_available: bool
|
||||
try:
|
||||
from PIL import Image # noqa: F401
|
||||
except ImportError:
|
||||
skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow"
|
||||
pillow_available = False
|
||||
else:
|
||||
skip_pillow = None
|
||||
pillow_available = True
|
||||
|
||||
|
||||
class ImageDownloadCrawlTestCase(TestFileDownloadCrawl):
|
||||
skip = skip_pillow
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not pillow_available,
|
||||
reason="Missing Python Imaging Library, install https://pypi.org/pypi/Pillow",
|
||||
)
|
||||
class TestImageDownloadCrawl(TestFileDownloadCrawl):
|
||||
pipeline_class = "scrapy.pipelines.images.ImagesPipeline"
|
||||
store_setting_key = "IMAGES_STORE"
|
||||
media_key = "images"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import os
|
|||
import random
|
||||
import time
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
|
@ -265,7 +266,12 @@ class TestFilesPipeline(unittest.TestCase):
|
|||
assert file_path(request, item=item) == "full/path-to-store-file"
|
||||
|
||||
|
||||
class FilesPipelineTestCaseFieldsMixin:
|
||||
class TestFilesPipelineFieldsMixin(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def item_class(self) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
def test_item_fields_default(self, tmp_path):
|
||||
url = "http://www.example.com/files/1.txt"
|
||||
item = self.item_class(name="item1", file_urls=[url])
|
||||
|
|
@ -302,7 +308,7 @@ class FilesPipelineTestCaseFieldsMixin:
|
|||
assert isinstance(item, self.item_class)
|
||||
|
||||
|
||||
class TestFilesPipelineFieldsDict(FilesPipelineTestCaseFieldsMixin):
|
||||
class TestFilesPipelineFieldsDict(TestFilesPipelineFieldsMixin):
|
||||
item_class = dict
|
||||
|
||||
|
||||
|
|
@ -316,7 +322,7 @@ class FilesPipelineTestItem(Item):
|
|||
custom_files = Field()
|
||||
|
||||
|
||||
class TestFilesPipelineFieldsItem(FilesPipelineTestCaseFieldsMixin):
|
||||
class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):
|
||||
item_class = FilesPipelineTestItem
|
||||
|
||||
|
||||
|
|
@ -331,7 +337,7 @@ class FilesPipelineTestDataClass:
|
|||
custom_files: list = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
class TestFilesPipelineFieldsDataClass(FilesPipelineTestCaseFieldsMixin):
|
||||
class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):
|
||||
item_class = FilesPipelineTestDataClass
|
||||
|
||||
|
||||
|
|
@ -346,7 +352,7 @@ class FilesPipelineTestAttrsItem:
|
|||
custom_files: list[dict[str, str]] = attr.ib(default=list)
|
||||
|
||||
|
||||
class TestFilesPipelineFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin):
|
||||
class TestFilesPipelineFieldsAttrsItem(TestFilesPipelineFieldsMixin):
|
||||
item_class = FilesPipelineTestAttrsItem
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ from __future__ import annotations
|
|||
import dataclasses
|
||||
import io
|
||||
import random
|
||||
from abc import ABC, abstractmethod
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
from typing import Any
|
||||
|
||||
import attr
|
||||
import pytest
|
||||
|
|
@ -208,7 +210,12 @@ class TestImagesPipeline:
|
|||
assert converted.getcolors() == [(10000, (205, 230, 255))]
|
||||
|
||||
|
||||
class ImagesPipelineTestCaseFieldsMixin:
|
||||
class TestImagesPipelineFieldsMixin(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def item_class(self) -> Any:
|
||||
raise NotImplementedError
|
||||
|
||||
def test_item_fields_default(self):
|
||||
url = "http://www.example.com/images/1.jpg"
|
||||
item = self.item_class(name="item1", image_urls=[url])
|
||||
|
|
@ -245,7 +252,7 @@ class ImagesPipelineTestCaseFieldsMixin:
|
|||
assert isinstance(item, self.item_class)
|
||||
|
||||
|
||||
class TestImagesPipelineFieldsDict(ImagesPipelineTestCaseFieldsMixin):
|
||||
class TestImagesPipelineFieldsDict(TestImagesPipelineFieldsMixin):
|
||||
item_class = dict
|
||||
|
||||
|
||||
|
|
@ -259,7 +266,7 @@ class ImagesPipelineTestItem(Item):
|
|||
custom_images = Field()
|
||||
|
||||
|
||||
class TestImagesPipelineFieldsItem(ImagesPipelineTestCaseFieldsMixin):
|
||||
class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin):
|
||||
item_class = ImagesPipelineTestItem
|
||||
|
||||
|
||||
|
|
@ -274,7 +281,7 @@ class ImagesPipelineTestDataClass:
|
|||
custom_images: list = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
class TestImagesPipelineFieldsDataClass(ImagesPipelineTestCaseFieldsMixin):
|
||||
class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin):
|
||||
item_class = ImagesPipelineTestDataClass
|
||||
|
||||
|
||||
|
|
@ -289,7 +296,7 @@ class ImagesPipelineTestAttrsItem:
|
|||
custom_images: list[dict[str, str]] = attr.ib(default=list)
|
||||
|
||||
|
||||
class TestImagesPipelineFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin):
|
||||
class TestImagesPipelineFieldsAttrsItem(TestImagesPipelineFieldsMixin):
|
||||
item_class = ImagesPipelineTestAttrsItem
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -345,7 +345,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
|
||||
@inlineCallbacks
|
||||
def test_use_media_to_download_result(self):
|
||||
req = Request("http://url", meta={"result": "ITSME", "response": self.fail})
|
||||
req = Request("http://url", meta={"result": "ITSME"})
|
||||
item = {"requests": req}
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
assert new_item["results"] == [(True, "ITSME")]
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ class TestMinimalScheduler(InterfaceCheckMixin):
|
|||
assert not self.scheduler.has_pending_requests()
|
||||
|
||||
|
||||
class SimpleSchedulerTest(TestCase, InterfaceCheckMixin):
|
||||
class TestSimpleScheduler(TestCase, InterfaceCheckMixin):
|
||||
def setUp(self):
|
||||
self.scheduler = SimpleScheduler()
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ class SimpleSchedulerTest(TestCase, InterfaceCheckMixin):
|
|||
assert close_result == "close"
|
||||
|
||||
|
||||
class MinimalSchedulerCrawlTest(TestCase):
|
||||
class TestMinimalSchedulerCrawl(TestCase):
|
||||
scheduler_cls = MinimalScheduler
|
||||
|
||||
@inlineCallbacks
|
||||
|
|
@ -162,5 +162,5 @@ class MinimalSchedulerCrawlTest(TestCase):
|
|||
assert f"'item_scraped_count': {len(PATHS)}" in str(log)
|
||||
|
||||
|
||||
class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest):
|
||||
class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl):
|
||||
scheduler_cls = SimpleScheduler
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ class ItemSpider(Spider):
|
|||
return {"index": response.meta["index"]}
|
||||
|
||||
|
||||
class MainTestCase(TestCase):
|
||||
class TestMain(TestCase):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_scheduler_empty(self):
|
||||
crawler = get_crawler()
|
||||
|
|
@ -35,7 +35,7 @@ class MainTestCase(TestCase):
|
|||
assert len(calls) >= 1
|
||||
|
||||
|
||||
class MockServerTestCase(TestCase):
|
||||
class TestMockServer(TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.mockserver = MockServer()
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ ITEM_A = {"id": "a"}
|
|||
ITEM_B = {"id": "b"}
|
||||
|
||||
|
||||
class MainTestCase(TestCase):
|
||||
class TestMain(TestCase):
|
||||
async def _test_spider(self, spider, expected_items=None):
|
||||
actual_items = []
|
||||
expected_items = [] if expected_items is None else expected_items
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class UniversalSpiderMiddleware:
|
|||
raise NotImplementedError
|
||||
|
||||
|
||||
# Spiders and spider middlewares for MainTestCase._test_wrap
|
||||
# Spiders and spider middlewares for TestMain._test_wrap
|
||||
|
||||
|
||||
class ModernWrapSpider(Spider):
|
||||
|
|
@ -106,7 +106,7 @@ class DeprecatedWrapSpiderMiddleware:
|
|||
yield ITEM_C
|
||||
|
||||
|
||||
class MainTestCase(TestCase):
|
||||
class TestMain(TestCase):
|
||||
async def _test(self, spider_middlewares, spider_cls, expected_items):
|
||||
actual_items = []
|
||||
|
||||
|
|
|
|||
|
|
@ -92,10 +92,10 @@ class TestDeferUtils(unittest.TestCase):
|
|||
x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
|
||||
assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]
|
||||
|
||||
@inlineCallbacks
|
||||
def test_process_parallel_failure(self):
|
||||
d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
|
||||
self.failUnlessFailure(d, TypeError)
|
||||
return d
|
||||
with pytest.raises(TypeError):
|
||||
yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
|
||||
|
||||
|
||||
class TestIterErrback:
|
||||
|
|
|
|||
|
|
@ -59,12 +59,12 @@ class TestSendCatchLog(unittest.TestCase):
|
|||
return "OK"
|
||||
|
||||
|
||||
class SendCatchLogDeferredTest(TestSendCatchLog):
|
||||
class TestSendCatchLogDeferred(TestSendCatchLog):
|
||||
def _get_result(self, signal, *a, **kw):
|
||||
return send_catch_log_deferred(signal, *a, **kw)
|
||||
|
||||
|
||||
class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
|
||||
class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):
|
||||
def ok_handler(self, arg, handlers_called):
|
||||
from twisted.internet import reactor
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("reactor_pytest")
|
||||
class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
|
||||
class TestSendCatchLogDeferredAsyncDef(TestSendCatchLogDeferred):
|
||||
async def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
assert arg == "test"
|
||||
|
|
@ -85,7 +85,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
|
|||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
|
||||
class TestSendCatchLogDeferredAsyncio(TestSendCatchLogDeferred):
|
||||
async def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
assert arg == "test"
|
||||
|
|
@ -93,12 +93,12 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
|
|||
return await get_from_asyncio_queue("OK")
|
||||
|
||||
|
||||
class SendCatchLogAsyncTest(TestSendCatchLog):
|
||||
class TestSendCatchLogAsync(TestSendCatchLog):
|
||||
def _get_result(self, signal, *a, **kw):
|
||||
return deferred_from_coro(send_catch_log_async(signal, *a, **kw))
|
||||
|
||||
|
||||
class SendCatchLogAsyncTest2(SendCatchLogAsyncTest):
|
||||
class TestSendCatchLogAsync2(TestSendCatchLogAsync):
|
||||
def ok_handler(self, arg, handlers_called):
|
||||
from twisted.internet import reactor
|
||||
|
||||
|
|
@ -110,7 +110,7 @@ class SendCatchLogAsyncTest2(SendCatchLogAsyncTest):
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("reactor_pytest")
|
||||
class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest):
|
||||
class TestSendCatchLogAsyncAsyncDef(TestSendCatchLogAsync):
|
||||
async def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
assert arg == "test"
|
||||
|
|
@ -119,7 +119,7 @@ class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest):
|
|||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class SendCatchLogAsyncAsyncioTest(SendCatchLogAsyncTest):
|
||||
class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync):
|
||||
async def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
assert arg == "test"
|
||||
|
|
|
|||
|
|
@ -234,35 +234,31 @@ class TestWebClient(unittest.TestCase):
|
|||
def getURL(self, path):
|
||||
return f"http://127.0.0.1:{self.portno}/{path}"
|
||||
|
||||
@inlineCallbacks
|
||||
def testPayload(self):
|
||||
s = "0123456789" * 10
|
||||
return getPage(self.getURL("payload"), body=s).addCallback(
|
||||
self.assertEqual, to_bytes(s)
|
||||
)
|
||||
body = yield getPage(self.getURL("payload"), body=s)
|
||||
assert body == to_bytes(s)
|
||||
|
||||
@inlineCallbacks
|
||||
def testHostHeader(self):
|
||||
# if we pass Host header explicitly, it should be used, otherwise
|
||||
# it should extract from url
|
||||
return defer.gatherResults(
|
||||
[
|
||||
getPage(self.getURL("host")).addCallback(
|
||||
self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")
|
||||
),
|
||||
getPage(
|
||||
self.getURL("host"), headers={"Host": "www.example.com"}
|
||||
).addCallback(self.assertEqual, to_bytes("www.example.com")),
|
||||
]
|
||||
)
|
||||
body = yield getPage(self.getURL("host"))
|
||||
assert body == to_bytes(f"127.0.0.1:{self.portno}")
|
||||
body = yield getPage(self.getURL("host"), headers={"Host": "www.example.com"})
|
||||
assert body == to_bytes("www.example.com")
|
||||
|
||||
@inlineCallbacks
|
||||
def test_getPage(self):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the body of the response if the default method B{GET} is used.
|
||||
"""
|
||||
d = getPage(self.getURL("file"))
|
||||
d.addCallback(self.assertEqual, b"0123456789")
|
||||
return d
|
||||
body = yield getPage(self.getURL("file"))
|
||||
assert body == b"0123456789"
|
||||
|
||||
@inlineCallbacks
|
||||
def test_getPageHead(self):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
|
|
@ -273,22 +269,20 @@ class TestWebClient(unittest.TestCase):
|
|||
def _getPage(method):
|
||||
return getPage(self.getURL("file"), method=method)
|
||||
|
||||
return defer.gatherResults(
|
||||
[
|
||||
_getPage("head").addCallback(self.assertEqual, b""),
|
||||
_getPage("HEAD").addCallback(self.assertEqual, b""),
|
||||
]
|
||||
)
|
||||
body = yield _getPage("head")
|
||||
assert body == b""
|
||||
body = yield _getPage("HEAD")
|
||||
assert body == b""
|
||||
|
||||
@inlineCallbacks
|
||||
def test_timeoutNotTriggering(self):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and the page is
|
||||
retrieved before the timeout period elapses, the L{Deferred} is
|
||||
called back with the contents of the page.
|
||||
"""
|
||||
d = getPage(self.getURL("host"), timeout=100)
|
||||
d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}"))
|
||||
return d
|
||||
body = yield getPage(self.getURL("host"), timeout=100)
|
||||
assert body == to_bytes(f"127.0.0.1:{self.portno}")
|
||||
|
||||
@inlineCallbacks
|
||||
def test_timeoutTriggering(self):
|
||||
|
|
@ -307,12 +301,12 @@ class TestWebClient(unittest.TestCase):
|
|||
if connected:
|
||||
connected[0].transport.loseConnection()
|
||||
|
||||
@inlineCallbacks
|
||||
def testNotFound(self):
|
||||
return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile)
|
||||
|
||||
def _cbNoSuchFile(self, pageData):
|
||||
assert b"404 - No Such Resource" in pageData
|
||||
body = yield getPage(self.getURL("notsuchfile"))
|
||||
assert b"404 - No Such Resource" in body
|
||||
|
||||
@inlineCallbacks
|
||||
def testFactoryInfo(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
|
|
@ -320,63 +314,60 @@ class TestWebClient(unittest.TestCase):
|
|||
parsed = urlparse(url)
|
||||
factory = client.ScrapyHTTPClientFactory(Request(url))
|
||||
reactor.connectTCP(parsed.hostname, parsed.port, factory)
|
||||
return factory.deferred.addCallback(self._cbFactoryInfo, factory)
|
||||
|
||||
def _cbFactoryInfo(self, ignoredResult, factory):
|
||||
yield factory.deferred
|
||||
assert factory.status == b"200"
|
||||
assert factory.version.startswith(b"HTTP/")
|
||||
assert factory.message == b"OK"
|
||||
assert factory.response_headers[b"content-length"] == b"10"
|
||||
|
||||
@inlineCallbacks
|
||||
def testRedirect(self):
|
||||
return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
|
||||
|
||||
def _cbRedirect(self, pageData):
|
||||
body = yield getPage(self.getURL("redirect"))
|
||||
assert (
|
||||
pageData
|
||||
body
|
||||
== b'\n<html>\n <head>\n <meta http-equiv="refresh" content="0;URL=/file">\n'
|
||||
b' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
|
||||
b'<a href="/file">click here</a>\n </body>\n</html>\n'
|
||||
)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_encoding(self):
|
||||
"""Test that non-standart body encoding matches
|
||||
Content-Encoding header"""
|
||||
body = b"\xd0\x81\xd1\x8e\xd0\xaf"
|
||||
dfd = getPage(
|
||||
self.getURL("encoding"), body=body, response_transform=lambda r: r
|
||||
original_body = b"\xd0\x81\xd1\x8e\xd0\xaf"
|
||||
response = yield getPage(
|
||||
self.getURL("encoding"), body=original_body, response_transform=lambda r: r
|
||||
)
|
||||
return dfd.addCallback(self._check_Encoding, body)
|
||||
|
||||
def _check_Encoding(self, response, original_body):
|
||||
content_encoding = to_unicode(response.headers[b"Content-Encoding"])
|
||||
assert content_encoding == EncodingResource.out_encoding
|
||||
assert response.body.decode(content_encoding) == to_unicode(original_body)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class WebClientSSLTestCase(TestContextFactoryBase):
|
||||
class TestWebClientSSL(TestContextFactoryBase):
|
||||
@inlineCallbacks
|
||||
def testPayload(self):
|
||||
s = "0123456789" * 10
|
||||
return getPage(self.getURL("payload"), body=s).addCallback(
|
||||
self.assertEqual, to_bytes(s)
|
||||
)
|
||||
body = yield getPage(self.getURL("payload"), body=s)
|
||||
assert body == to_bytes(s)
|
||||
|
||||
|
||||
class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
|
||||
class TestWebClientCustomCiphersSSL(TestWebClientSSL):
|
||||
# we try to use a cipher that is not enabled by default in OpenSSL
|
||||
custom_ciphers = "CAMELLIA256-SHA"
|
||||
context_factory = ssl_context_factory(cipher_string=custom_ciphers)
|
||||
|
||||
@inlineCallbacks
|
||||
def testPayload(self):
|
||||
s = "0123456789" * 10
|
||||
crawler = get_crawler(
|
||||
settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}
|
||||
)
|
||||
client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler)
|
||||
return getPage(
|
||||
body = yield getPage(
|
||||
self.getURL("payload"), body=s, contextFactory=client_context_factory
|
||||
).addCallback(self.assertEqual, to_bytes(s))
|
||||
)
|
||||
assert body == to_bytes(s)
|
||||
|
||||
@inlineCallbacks
|
||||
def testPayloadDisabledCipher(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue