mirror of https://github.com/scrapy/scrapy.git
660 lines
23 KiB
Python
660 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import random
|
|
import re
|
|
import shutil
|
|
import string
|
|
from ipaddress import IPv4Address
|
|
from pathlib import Path
|
|
from tempfile import mkdtemp
|
|
from typing import TYPE_CHECKING, Any, Callable
|
|
from unittest import mock
|
|
from urllib.parse import urlencode
|
|
|
|
import pytest
|
|
from twisted.internet.defer import (
|
|
CancelledError,
|
|
Deferred,
|
|
DeferredList,
|
|
inlineCallbacks,
|
|
)
|
|
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
|
|
from twisted.internet.error import TimeoutError
|
|
from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS
|
|
from twisted.trial.unittest import TestCase
|
|
from twisted.web.client import URI, ResponseFailed
|
|
from twisted.web.http import H2_ENABLED
|
|
from twisted.web.http import Request as TxRequest
|
|
from twisted.web.server import NOT_DONE_YET, Site
|
|
from twisted.web.static import File
|
|
|
|
from scrapy.http import JsonRequest, Request, Response
|
|
from scrapy.settings import Settings
|
|
from scrapy.spiders import Spider
|
|
from scrapy.utils.defer import (
|
|
deferred_f_from_coro_f,
|
|
deferred_from_coro,
|
|
maybe_deferred_to_future,
|
|
)
|
|
from tests.mockserver import LeafResource, Status, ssl_context_factory
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Coroutine
|
|
|
|
|
|
def generate_random_string(size: int) -> str:
|
|
return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))
|
|
|
|
|
|
def make_html_body(val: str) -> bytes:
|
|
response = f"""<html>
|
|
<h1>Hello from HTTP2<h1>
|
|
<p>{val}</p>
|
|
</html>"""
|
|
return bytes(response, "utf-8")
|
|
|
|
|
|
class DummySpider(Spider):
|
|
name = "dummy"
|
|
start_urls: list = []
|
|
|
|
def parse(self, response):
|
|
print(response)
|
|
|
|
|
|
class Data:
|
|
SMALL_SIZE = 1024 # 1 KB
|
|
LARGE_SIZE = 1024**2 # 1 MB
|
|
|
|
STR_SMALL = generate_random_string(SMALL_SIZE)
|
|
STR_LARGE = generate_random_string(LARGE_SIZE)
|
|
|
|
EXTRA_SMALL = generate_random_string(1024 * 15)
|
|
EXTRA_LARGE = generate_random_string((1024**2) * 15)
|
|
|
|
HTML_SMALL = make_html_body(STR_SMALL)
|
|
HTML_LARGE = make_html_body(STR_LARGE)
|
|
|
|
JSON_SMALL = {"data": STR_SMALL}
|
|
JSON_LARGE = {"data": STR_LARGE}
|
|
|
|
DATALOSS = b"Dataloss Content"
|
|
NO_CONTENT_LENGTH = b"This response do not have any content-length header"
|
|
|
|
|
|
class GetDataHtmlSmall(LeafResource):
|
|
def render_GET(self, request: TxRequest):
|
|
request.setHeader("Content-Type", "text/html; charset=UTF-8")
|
|
return Data.HTML_SMALL
|
|
|
|
|
|
class GetDataHtmlLarge(LeafResource):
|
|
def render_GET(self, request: TxRequest):
|
|
request.setHeader("Content-Type", "text/html; charset=UTF-8")
|
|
return Data.HTML_LARGE
|
|
|
|
|
|
class PostDataJsonMixin:
|
|
@staticmethod
|
|
def make_response(request: TxRequest, extra_data: str) -> bytes:
|
|
assert request.content is not None
|
|
response = {
|
|
"request-headers": {},
|
|
"request-body": json.loads(request.content.read()),
|
|
"extra-data": extra_data,
|
|
}
|
|
for k, v in request.requestHeaders.getAllRawHeaders():
|
|
response["request-headers"][str(k, "utf-8")] = str(v[0], "utf-8")
|
|
|
|
response_bytes = bytes(json.dumps(response), "utf-8")
|
|
request.setHeader("Content-Type", "application/json; charset=UTF-8")
|
|
request.setHeader("Content-Encoding", "UTF-8")
|
|
return response_bytes
|
|
|
|
|
|
class PostDataJsonSmall(LeafResource, PostDataJsonMixin):
|
|
def render_POST(self, request: TxRequest):
|
|
return self.make_response(request, Data.EXTRA_SMALL)
|
|
|
|
|
|
class PostDataJsonLarge(LeafResource, PostDataJsonMixin):
|
|
def render_POST(self, request: TxRequest):
|
|
return self.make_response(request, Data.EXTRA_LARGE)
|
|
|
|
|
|
class Dataloss(LeafResource):
|
|
def render_GET(self, request: TxRequest):
|
|
request.setHeader(b"Content-Length", b"1024")
|
|
self.deferRequest(request, 0, self._delayed_render, request)
|
|
return NOT_DONE_YET
|
|
|
|
@staticmethod
|
|
def _delayed_render(request: TxRequest):
|
|
request.write(Data.DATALOSS)
|
|
request.finish()
|
|
|
|
|
|
class NoContentLengthHeader(LeafResource):
|
|
def render_GET(self, request: TxRequest):
|
|
request.requestHeaders.removeHeader("Content-Length")
|
|
self.deferRequest(request, 0, self._delayed_render, request)
|
|
return NOT_DONE_YET
|
|
|
|
@staticmethod
|
|
def _delayed_render(request: TxRequest):
|
|
request.write(Data.NO_CONTENT_LENGTH)
|
|
request.finish()
|
|
|
|
|
|
class TimeoutResponse(LeafResource):
|
|
def render_GET(self, request: TxRequest):
|
|
return NOT_DONE_YET
|
|
|
|
|
|
class QueryParams(LeafResource):
|
|
def render_GET(self, request: TxRequest):
|
|
request.setHeader("Content-Type", "application/json; charset=UTF-8")
|
|
request.setHeader("Content-Encoding", "UTF-8")
|
|
|
|
query_params: dict[str, str] = {}
|
|
assert request.args is not None
|
|
for k, v in request.args.items():
|
|
query_params[str(k, "utf-8")] = str(v[0], "utf-8")
|
|
|
|
return bytes(json.dumps(query_params), "utf-8")
|
|
|
|
|
|
class RequestHeaders(LeafResource):
|
|
"""Sends all the headers received as a response"""
|
|
|
|
def render_GET(self, request: TxRequest):
|
|
request.setHeader("Content-Type", "application/json; charset=UTF-8")
|
|
request.setHeader("Content-Encoding", "UTF-8")
|
|
headers = {}
|
|
for k, v in request.requestHeaders.getAllRawHeaders():
|
|
headers[str(k, "utf-8")] = str(v[0], "utf-8")
|
|
|
|
return bytes(json.dumps(headers), "utf-8")
|
|
|
|
|
|
def get_client_certificate(
|
|
key_file: Path, certificate_file: Path
|
|
) -> PrivateCertificate:
|
|
pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(
|
|
encoding="utf-8"
|
|
)
|
|
return PrivateCertificate.loadPEM(pem)
|
|
|
|
|
|
@pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled")
|
|
class TestHttps2ClientProtocol(TestCase):
|
|
scheme = "https"
|
|
key_file = Path(__file__).parent / "keys" / "localhost.key"
|
|
certificate_file = Path(__file__).parent / "keys" / "localhost.crt"
|
|
|
|
def _init_resource(self):
|
|
self.temp_directory = mkdtemp()
|
|
r = File(self.temp_directory)
|
|
r.putChild(b"get-data-html-small", GetDataHtmlSmall())
|
|
r.putChild(b"get-data-html-large", GetDataHtmlLarge())
|
|
|
|
r.putChild(b"post-data-json-small", PostDataJsonSmall())
|
|
r.putChild(b"post-data-json-large", PostDataJsonLarge())
|
|
|
|
r.putChild(b"dataloss", Dataloss())
|
|
r.putChild(b"no-content-length-header", NoContentLengthHeader())
|
|
r.putChild(b"status", Status())
|
|
r.putChild(b"query-params", QueryParams())
|
|
r.putChild(b"timeout", TimeoutResponse())
|
|
r.putChild(b"request-headers", RequestHeaders())
|
|
return r
|
|
|
|
@inlineCallbacks
|
|
def setUp(self):
|
|
from twisted.internet import reactor
|
|
|
|
# Initialize resource tree
|
|
root = self._init_resource()
|
|
self.site = Site(root, timeout=None)
|
|
|
|
# Start server for testing
|
|
self.hostname = "localhost"
|
|
context_factory = ssl_context_factory(
|
|
str(self.key_file), str(self.certificate_file)
|
|
)
|
|
|
|
server_endpoint = SSL4ServerEndpoint(
|
|
reactor, 0, context_factory, interface=self.hostname
|
|
)
|
|
self.server = yield server_endpoint.listen(self.site)
|
|
self.port_number = self.server.getHost().port
|
|
|
|
# Connect H2 client with server
|
|
self.client_certificate = get_client_certificate(
|
|
self.key_file, self.certificate_file
|
|
)
|
|
client_options = optionsForClientTLS(
|
|
hostname=self.hostname,
|
|
trustRoot=self.client_certificate,
|
|
acceptableProtocols=[b"h2"],
|
|
)
|
|
uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8"))
|
|
|
|
self.conn_closed_deferred = Deferred()
|
|
|
|
from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415
|
|
|
|
h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
|
|
client_endpoint = SSL4ClientEndpoint(
|
|
reactor, self.hostname, self.port_number, client_options
|
|
)
|
|
self.client = yield client_endpoint.connect(h2_client_factory)
|
|
|
|
@inlineCallbacks
|
|
def tearDown(self):
|
|
if self.client.connected:
|
|
yield self.client.transport.loseConnection()
|
|
yield self.client.transport.abortConnection()
|
|
yield self.server.stopListening()
|
|
shutil.rmtree(self.temp_directory)
|
|
self.conn_closed_deferred = None
|
|
|
|
def get_url(self, path: str) -> str:
|
|
"""
|
|
:param path: Should have / at the starting compulsorily if not empty
|
|
:return: Complete url
|
|
"""
|
|
assert len(path) > 0
|
|
assert path[0] == "/" or path[0] == "&"
|
|
return f"{self.scheme}://{self.hostname}:{self.port_number}{path}"
|
|
|
|
async def make_request(self, request: Request) -> Response:
|
|
return await maybe_deferred_to_future(self.make_request_dfd(request))
|
|
|
|
def make_request_dfd(self, request: Request) -> Deferred[Response]:
|
|
return self.client.request(request, DummySpider())
|
|
|
|
@staticmethod
|
|
async def _check_repeat(
|
|
get_coro: Callable[[], Coroutine[Any, Any, None]], count: int
|
|
) -> None:
|
|
d_list = []
|
|
for _ in range(count):
|
|
d = deferred_from_coro(get_coro())
|
|
d_list.append(d)
|
|
|
|
await maybe_deferred_to_future(DeferredList(d_list, fireOnOneErrback=True))
|
|
|
|
async def _check_GET(
|
|
self, request: Request, expected_body: bytes, expected_status: int
|
|
) -> None:
|
|
response = await self.make_request(request)
|
|
assert response.status == expected_status
|
|
assert response.body == expected_body
|
|
assert response.request == request
|
|
|
|
content_length_header = response.headers.get("Content-Length")
|
|
assert content_length_header is not None
|
|
content_length = int(content_length_header)
|
|
assert len(response.body) == content_length
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_GET_small_body(self):
|
|
request = Request(self.get_url("/get-data-html-small"))
|
|
await self._check_GET(request, Data.HTML_SMALL, 200)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_GET_large_body(self):
|
|
request = Request(self.get_url("/get-data-html-large"))
|
|
await self._check_GET(request, Data.HTML_LARGE, 200)
|
|
|
|
async def _check_GET_x10(
|
|
self, request: Request, expected_body: bytes, expected_status: int
|
|
) -> None:
|
|
async def get_coro() -> None:
|
|
await self._check_GET(request, expected_body, expected_status)
|
|
|
|
await self._check_repeat(get_coro, 10)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_GET_small_body_x10(self):
|
|
await self._check_GET_x10(
|
|
Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200
|
|
)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_GET_large_body_x10(self):
|
|
await self._check_GET_x10(
|
|
Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200
|
|
)
|
|
|
|
async def _check_POST_json(
|
|
self,
|
|
request: Request,
|
|
expected_request_body: dict[str, str],
|
|
expected_extra_data: str,
|
|
expected_status: int,
|
|
) -> None:
|
|
response = await self.make_request(request)
|
|
|
|
assert response.status == expected_status
|
|
assert response.request == request
|
|
|
|
content_length_header = response.headers.get("Content-Length")
|
|
assert content_length_header is not None
|
|
content_length = int(content_length_header)
|
|
assert len(response.body) == content_length
|
|
|
|
# Parse the body
|
|
content_encoding_header = response.headers[b"Content-Encoding"]
|
|
assert content_encoding_header is not None
|
|
content_encoding = str(content_encoding_header, "utf-8")
|
|
body = json.loads(str(response.body, content_encoding))
|
|
assert "request-body" in body
|
|
assert "extra-data" in body
|
|
assert "request-headers" in body
|
|
|
|
request_body = body["request-body"]
|
|
assert request_body == expected_request_body
|
|
|
|
extra_data = body["extra-data"]
|
|
assert extra_data == expected_extra_data
|
|
|
|
# Check if headers were sent successfully
|
|
request_headers = body["request-headers"]
|
|
for k, v in request.headers.items():
|
|
k_str = str(k, "utf-8")
|
|
assert k_str in request_headers
|
|
assert request_headers[k_str] == str(v[0], "utf-8")
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_POST_small_json(self):
|
|
request = JsonRequest(
|
|
url=self.get_url("/post-data-json-small"),
|
|
method="POST",
|
|
data=Data.JSON_SMALL,
|
|
)
|
|
await self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_POST_large_json(self):
|
|
request = JsonRequest(
|
|
url=self.get_url("/post-data-json-large"),
|
|
method="POST",
|
|
data=Data.JSON_LARGE,
|
|
)
|
|
await self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200)
|
|
|
|
async def _check_POST_json_x10(self, *args, **kwargs):
|
|
async def get_coro() -> None:
|
|
await self._check_POST_json(*args, **kwargs)
|
|
|
|
await self._check_repeat(get_coro, 10)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_POST_small_json_x10(self):
|
|
request = JsonRequest(
|
|
url=self.get_url("/post-data-json-small"),
|
|
method="POST",
|
|
data=Data.JSON_SMALL,
|
|
)
|
|
await self._check_POST_json_x10(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_POST_large_json_x10(self):
|
|
request = JsonRequest(
|
|
url=self.get_url("/post-data-json-large"),
|
|
method="POST",
|
|
data=Data.JSON_LARGE,
|
|
)
|
|
await self._check_POST_json_x10(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200)
|
|
|
|
@inlineCallbacks
|
|
def test_invalid_negotiated_protocol(self):
|
|
with mock.patch(
|
|
"scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
|
|
):
|
|
request = Request(url=self.get_url("/status?n=200"))
|
|
with pytest.raises(ResponseFailed):
|
|
yield self.make_request_dfd(request)
|
|
|
|
@inlineCallbacks
|
|
def test_cancel_request(self):
|
|
request = Request(url=self.get_url("/get-data-html-large"))
|
|
d = self.make_request_dfd(request)
|
|
d.cancel()
|
|
response = yield d
|
|
assert response.status == 499
|
|
assert response.request == request
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_download_maxsize_exceeded(self):
|
|
request = Request(
|
|
url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000}
|
|
)
|
|
with pytest.raises(CancelledError) as exc_info:
|
|
await self.make_request(request)
|
|
error_pattern = re.compile(
|
|
rf"Cancelling download of {request.url}: received response "
|
|
rf"size \(\d*\) larger than download max size \(1000\)"
|
|
)
|
|
assert len(re.findall(error_pattern, str(exc_info.value))) == 1
|
|
|
|
@inlineCallbacks
|
|
def test_received_dataloss_response(self):
|
|
"""In case when value of Header Content-Length != len(Received Data)
|
|
ProtocolError is raised"""
|
|
from h2.exceptions import InvalidBodyLengthError # noqa: PLC0415
|
|
|
|
request = Request(url=self.get_url("/dataloss"))
|
|
with pytest.raises(ResponseFailed) as exc_info:
|
|
yield self.make_request_dfd(request)
|
|
assert len(exc_info.value.reasons) > 0
|
|
assert any(
|
|
isinstance(error, InvalidBodyLengthError)
|
|
for error in exc_info.value.reasons
|
|
)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_missing_content_length_header(self):
|
|
request = Request(url=self.get_url("/no-content-length-header"))
|
|
response = await self.make_request(request)
|
|
assert response.status == 200
|
|
assert response.body == Data.NO_CONTENT_LENGTH
|
|
assert response.request == request
|
|
assert "Content-Length" not in response.headers
|
|
|
|
async def _check_log_warnsize(
|
|
self, request: Request, warn_pattern: re.Pattern[str], expected_body: bytes
|
|
) -> None:
|
|
with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm:
|
|
response = await self.make_request(request)
|
|
assert response.status == 200
|
|
assert response.request == request
|
|
assert response.body == expected_body
|
|
|
|
# Check the warning is raised only once for this request
|
|
assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_log_expected_warnsize(self):
|
|
request = Request(
|
|
url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000}
|
|
)
|
|
warn_pattern = re.compile(
|
|
rf"Expected response size \(\d*\) larger than "
|
|
rf"download warn size \(1000\) in request {request}"
|
|
)
|
|
|
|
await self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_log_received_warnsize(self):
|
|
request = Request(
|
|
url=self.get_url("/no-content-length-header"),
|
|
meta={"download_warnsize": 10},
|
|
)
|
|
warn_pattern = re.compile(
|
|
rf"Received more \(\d*\) bytes than download "
|
|
rf"warn size \(10\) in request {request}"
|
|
)
|
|
|
|
await self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_max_concurrent_streams(self):
|
|
"""Send 500 requests at one to check if we can handle
|
|
very large number of request.
|
|
"""
|
|
|
|
async def get_coro() -> None:
|
|
await self._check_GET(
|
|
Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200
|
|
)
|
|
|
|
await self._check_repeat(get_coro, 500)
|
|
|
|
@inlineCallbacks
|
|
def test_inactive_stream(self):
|
|
"""Here we send 110 requests considering the MAX_CONCURRENT_STREAMS
|
|
by default is 100. After sending the first 100 requests we close the
|
|
connection."""
|
|
d_list = []
|
|
|
|
def assert_inactive_stream(failure):
|
|
assert failure.check(ResponseFailed) is not None
|
|
|
|
from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415
|
|
|
|
assert any(
|
|
isinstance(e, InactiveStreamClosed) for e in failure.value.reasons
|
|
)
|
|
|
|
# Send 100 request (we do not check the result)
|
|
for _ in range(100):
|
|
d = self.make_request_dfd(Request(self.get_url("/get-data-html-small")))
|
|
d.addBoth(lambda _: None)
|
|
d_list.append(d)
|
|
|
|
# Now send 10 extra request and save the response deferred in a list
|
|
for _ in range(10):
|
|
d = self.make_request_dfd(Request(self.get_url("/get-data-html-small")))
|
|
d.addCallback(lambda _: pytest.fail("This request should have failed"))
|
|
d.addErrback(assert_inactive_stream)
|
|
d_list.append(d)
|
|
|
|
# Close the connection now to fire all the extra 10 requests errback
|
|
# with InactiveStreamClosed
|
|
self.client.transport.loseConnection()
|
|
|
|
yield DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_invalid_request_type(self):
|
|
with pytest.raises(TypeError):
|
|
await self.make_request("https://InvalidDataTypePassed.com")
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_query_parameters(self):
|
|
params = {
|
|
"a": generate_random_string(20),
|
|
"b": generate_random_string(20),
|
|
"c": generate_random_string(20),
|
|
"d": generate_random_string(20),
|
|
}
|
|
request = Request(self.get_url(f"/query-params?{urlencode(params)}"))
|
|
response = await self.make_request(request)
|
|
content_encoding_header = response.headers[b"Content-Encoding"]
|
|
assert content_encoding_header is not None
|
|
content_encoding = str(content_encoding_header, "utf-8")
|
|
data = json.loads(str(response.body, content_encoding))
|
|
assert data == params
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_status_codes(self):
|
|
for status in [200, 404]:
|
|
request = Request(self.get_url(f"/status?n={status}"))
|
|
response = await self.make_request(request)
|
|
assert response.status == status
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_response_has_correct_certificate_ip_address(self):
|
|
request = Request(self.get_url("/status?n=200"))
|
|
response = await self.make_request(request)
|
|
assert response.request == request
|
|
assert isinstance(response.certificate, Certificate)
|
|
assert response.certificate.original is not None
|
|
assert response.certificate.getIssuer() == self.client_certificate.getIssuer()
|
|
assert response.certificate.getPublicKey().matches(
|
|
self.client_certificate.getPublicKey()
|
|
)
|
|
assert isinstance(response.ip_address, IPv4Address)
|
|
assert str(response.ip_address) == "127.0.0.1"
|
|
|
|
async def _check_invalid_netloc(self, url: str) -> None:
|
|
from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415
|
|
|
|
request = Request(url)
|
|
with pytest.raises(InvalidHostname) as exc_info:
|
|
await self.make_request(request)
|
|
error_msg = str(exc_info.value)
|
|
assert "localhost" in error_msg
|
|
assert "127.0.0.1" in error_msg
|
|
assert str(request) in error_msg
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_invalid_hostname(self):
|
|
await self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain")
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_invalid_host_port(self):
|
|
port = self.port_number + 1
|
|
await self._check_invalid_netloc(f"https://127.0.0.1:{port}")
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_connection_stays_with_invalid_requests(self):
|
|
await maybe_deferred_to_future(self.test_invalid_hostname())
|
|
await maybe_deferred_to_future(self.test_invalid_host_port())
|
|
await maybe_deferred_to_future(self.test_GET_small_body())
|
|
await maybe_deferred_to_future(self.test_POST_small_json())
|
|
|
|
@inlineCallbacks
|
|
def test_connection_timeout(self):
|
|
request = Request(self.get_url("/timeout"))
|
|
|
|
# Update the timer to 1s to test connection timeout
|
|
self.client.setTimeout(1)
|
|
|
|
with pytest.raises(ResponseFailed) as exc_info:
|
|
yield self.make_request_dfd(request)
|
|
|
|
for err in exc_info.value.reasons:
|
|
from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415
|
|
|
|
if isinstance(err, TimeoutError):
|
|
assert (
|
|
f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"
|
|
in str(err)
|
|
)
|
|
break
|
|
else:
|
|
pytest.fail("No TimeoutError raised.")
|
|
|
|
@deferred_f_from_coro_f
|
|
async def test_request_headers_received(self):
|
|
request = Request(
|
|
self.get_url("/request-headers"),
|
|
headers={"header-1": "header value 1", "header-2": "header value 2"},
|
|
)
|
|
response = await self.make_request(request)
|
|
assert response.status == 200
|
|
assert response.request == request
|
|
|
|
response_headers = json.loads(str(response.body, "utf-8"))
|
|
assert isinstance(response_headers, dict)
|
|
for k, v in request.headers.items():
|
|
k, v = str(k, "utf-8"), str(v[0], "utf-8")
|
|
assert k in response_headers
|
|
assert v == response_headers[k]
|