diff --git a/pyproject.toml b/pyproject.toml index d96790873..7ce2f2e4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,10 +125,8 @@ ignore_errors = true # deprecated modules [[tool.mypy.overrides]] module = [ - "scrapy.core.downloader.webclient", "scrapy.spiders.init", "scrapy.utils.testsite", - "tests.test_webclient", ] allow_any_generics = true allow_untyped_calls = true diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 19fc77959..bbda7efcd 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -114,10 +114,10 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): acceptableCiphers=self.tls_ciphers, ) - # kept for old-style HTTP/1.0 downloader context twisted calls, - # e.g. connectSSL() # should be removed together with ScrapyClientContextFactory - def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: + def getContext( + self, hostname: Any = None, port: Any = None + ) -> SSL.Context: # pragma: no cover return self._get_context() def _get_context(self) -> SSL.Context: diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index a7b592d00..dc1b1e375 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,7 +1,6 @@ # pragma: no file cover import warnings -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) @@ -16,6 +15,5 @@ warnings.warn( ) __all__ = [ - "HTTP10DownloadHandler", "HTTPDownloadHandler", ] diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py deleted file mode 100644 index f5d1bbd76..000000000 --- a/scrapy/core/downloader/handlers/http10.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Download handlers for http and https schemes""" - -from __future__ import annotations - -import warnings -from typing import TYPE_CHECKING - -from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import to_unicode - -if TYPE_CHECKING: - from twisted.internet.interfaces import IConnector - - # typing.Self requires Python 3.11 - from typing_extensions import Self - - from scrapy import Request - from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory - from scrapy.crawler import Crawler - from scrapy.http import Response - from scrapy.settings import BaseSettings - - -class HTTP10DownloadHandler: - lazy = False - - def __init__(self, settings: BaseSettings, crawler: Crawler): - warnings.warn( - "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): # pragma: no cover - raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") - self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( - settings["DOWNLOADER_HTTPCLIENTFACTORY"] - ) - if settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": - self.ClientContextFactory: type[_ScrapyClientContextFactory] = ( - _ScrapyClientContextFactory - ) - else: # pragma: no cover - warnings.warn( - "The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.ClientContextFactory = load_object( - settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] - ) - self._settings: BaseSettings = settings - self._crawler: Crawler = crawler - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings, crawler) - - async def download_request(self, request: Request) -> Response: - factory = self.HTTPClientFactory(request) - self._connect(factory) - return await maybe_deferred_to_future(factory.deferred) - - def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: - from twisted.internet import reactor - - host, port = to_unicode(factory.host), factory.port - if factory.scheme == b"https": - client_context_factory = build_from_crawler( - self.ClientContextFactory, - self._crawler, - ) - return reactor.connectSSL(host, port, factory, client_context_factory) - return reactor.connectTCP(host, port, factory) - - async def close(self) -> None: - pass diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py deleted file mode 100644 index 2a550cf78..000000000 --- a/scrapy/core/downloader/webclient.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler.""" - -from __future__ import annotations - -import warnings -from time import monotonic, time -from typing import TYPE_CHECKING -from urllib.parse import urldefrag, urlparse, urlunparse - -from twisted.internet import defer -from twisted.internet.protocol import ClientFactory -from twisted.web.http import HTTPClient - -from scrapy.exceptions import DownloadTimeoutError, ScrapyDeprecationWarning -from scrapy.http import Headers, Response -from scrapy.responsetypes import responsetypes -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode - -if TYPE_CHECKING: - from scrapy import Request - - -class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b"\n" - - def __init__(self): - warnings.warn( - "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__() - - def connectionMade(self): - self.headers = Headers() # bucket for response headers - - # Method command - self.sendCommand(self.factory.method, self.factory.path) - # Headers - for key, values in self.factory.headers.items(): - for value in values: - self.sendHeader(key, value) - self.endHeaders() - # Body - if self.factory.body is not None: - self.transport.write(self.factory.body) - - def lineReceived(self, line): - return HTTPClient.lineReceived(self, line.rstrip()) - - def handleHeader(self, key, value): - self.headers.appendlist(key, value) - - def handleStatus(self, version, status, message): - self.factory.gotStatus(version, status, message) - - def handleEndHeaders(self): - self.factory.gotHeaders(self.headers) - - def connectionLost(self, reason): - self._connection_lost_reason = reason - HTTPClient.connectionLost(self, reason) - self.factory.noPage(reason) - - def handleResponse(self, response): - if self.factory.method.upper() == b"HEAD": - self.factory.page(b"") - elif self.length is not None and self.length > 0: - self.factory.noPage(self._connection_lost_reason) - else: - self.factory.page(response) - self.transport.loseConnection() - - def timeout(self): - self.transport.loseConnection() - - # transport cleanup needed for HTTPS connections - if self.factory.url.startswith(b"https"): - self.transport.stopProducing() - - self.factory.noPage( - DownloadTimeoutError( - f"Getting {self.factory.url} took longer " - f"than {self.factory.timeout} seconds." - ) - ) - - -# This class used to inherit from Twisted’s -# twisted.web.client.HTTPClientFactory. When that class was deprecated in -# Twisted (https://github.com/twisted/twisted/pull/643), we merged its -# non-overridden code into this class. -class ScrapyHTTPClientFactory(ClientFactory): - protocol = ScrapyHTTPPageGetter - - waiting = 1 - noisy = False - followRedirect = False - afterFoundGet = False - - def _build_response(self, body, request): - request.meta["download_latency"] = ( - self._headers_time_mono - self._start_time_mono - ) - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) - return respcls( - url=self._url, - status=status, - headers=headers, - body=body, - protocol=to_unicode(self.version), - ) - - def _set_connection_attributes(self, request): - proxy = request.meta.get("proxy") - if proxy: - proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii")) - self.scheme = proxy_parsed.scheme - self.host = proxy_parsed.hostname - self.port = proxy_parsed.port - self.netloc = proxy_parsed.netloc - if self.port is None: - self.port = 443 if proxy_parsed.scheme == b"https" else 80 - self.path = self.url - else: - parsed = urlparse_cached(request) - path_str = urlunparse( - ("", "", parsed.path or "/", parsed.params, parsed.query, "") - ) - self.path = to_bytes(path_str, encoding="ascii") - assert parsed.hostname is not None - self.host = to_bytes(parsed.hostname, encoding="ascii") - self.port = parsed.port - self.scheme = to_bytes(parsed.scheme, encoding="ascii") - self.netloc = to_bytes(parsed.netloc, encoding="ascii") - if self.port is None: - self.port = 443 if self.scheme == b"https" else 80 - - def __init__(self, request: Request, timeout: float = 180): - warnings.warn( - "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - self._url: str = urldefrag(request.url)[0] - # converting to bytes to comply to Twisted interface - self.url: bytes = to_bytes(self._url, encoding="ascii") - self.method: bytes = to_bytes(request.method, encoding="ascii") - self.body: bytes | None = request.body or None - self.headers: Headers = Headers(request.headers) - self.response_headers: Headers | None = None - self.timeout: float = request.meta.get("download_timeout") or timeout - self.start_time: float = time() - self._start_time_mono: float = monotonic() - self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( - self._build_response, request - ) - - # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected - # to have _disconnectedDeferred. See Twisted r32329. - # As Scrapy implements it's own logic to handle redirects is not - # needed to add the callback _waitForDisconnect. - # Specifically this avoids the AttributeError exception when - # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred() - - self._set_connection_attributes(request) - - # set Host header based on url - self.headers.setdefault("Host", self.netloc) - - # set Content-Length based len of body - if self.body is not None: - self.headers["Content-Length"] = len(self.body) - # just in case a broken http/1.1 decides to keep connection alive - self.headers.setdefault("Connection", "close") - # Content-Length must be specified in POST method even with no body - elif self.method == b"POST": - self.headers["Content-Length"] = 0 - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}: {self._url}>" - - def _cancelTimeout(self, result, timeoutCall): - if timeoutCall.active(): - timeoutCall.cancel() - return result - - def buildProtocol(self, addr): - p = ClientFactory.buildProtocol(self, addr) - p.followRedirect = self.followRedirect - p.afterFoundGet = self.afterFoundGet - if self.timeout: - from twisted.internet import reactor - - timeoutCall = reactor.callLater(self.timeout, p.timeout) - self.deferred.addBoth(self._cancelTimeout, timeoutCall) - return p - - def gotHeaders(self, headers): - self.headers_time = time() - self._headers_time_mono = monotonic() - self.response_headers = headers - - def gotStatus(self, version, status, message): - """ - Set the status of the request on us. - @param version: The HTTP version. - @type version: L{bytes} - @param status: The HTTP status code, an integer represented as a - bytestring. - @type status: L{bytes} - @param message: The HTTP status message. - @type message: L{bytes} - """ - self.version, self.status, self.message = version, status, message - - def page(self, page): - if self.waiting: - self.waiting = 0 - self.deferred.callback(page) - - def noPage(self, reason): - if self.waiting: - self.waiting = 0 - self.deferred.errback(reason) - - def clientConnectionFailed(self, _, reason): - """ - When a connection attempt fails, the request cannot be issued. If no - result has yet been provided to the result Deferred, provide the - connection failure reason as an error result. - """ - if self.waiting: - self.waiting = 0 - # If the connection attempt failed, there is nothing more to - # disconnect, so just fire that Deferred now. - self._disconnectedDeferred.callback(None) - self.deferred.errback(reason) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b80e48601..f9297d4cf 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -54,7 +54,6 @@ __all__ = [ "DOWNLOADER_CLIENT_TLS_CIPHERS", "DOWNLOADER_CLIENT_TLS_METHOD", "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING", - "DOWNLOADER_HTTPCLIENTFACTORY", "DOWNLOADER_MIDDLEWARES", "DOWNLOADER_MIDDLEWARES_BASE", "DOWNLOADER_STATS", @@ -275,10 +274,6 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False -DOWNLOADER_HTTPCLIENTFACTORY = ( - "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" -) - DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 3b1796af7..a0e3fccb3 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -9,9 +9,9 @@ import pytest from scrapy import Request from tests.test_downloader_handlers_http_base import ( - TestHttp11Base, + TestHttpBase, TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, @@ -42,7 +42,7 @@ class HttpxDownloadHandlerMixin: return HttpxDownloadHandler -class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): +class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): handler_supports_bindaddress_meta = False @pytest.mark.skipif( @@ -77,7 +77,7 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): ) -class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base): +class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): handler_supports_bindaddress_meta = False tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" @@ -90,25 +90,25 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): pass -class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class TestHttps11InvalidDNSPattern( +class TestHttpsInvalidDNSPattern( HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class TestHttp11WithCrawler(TestHttpWithCrawlerBase): +class TestHttpWithCrawler(TestHttpWithCrawlerBase): @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -119,7 +119,7 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase): } -class TestHttps11WithCrawler(TestHttp11WithCrawler): +class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True @pytest.mark.skip(reason="response.certificate is not implemented") @@ -129,10 +129,10 @@ class TestHttps11WithCrawler(TestHttp11WithCrawler): @pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): pass @pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): is_secure = True diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py deleted file mode 100644 index 8281337b0..000000000 --- a/tests/test_downloader_handler_twisted_http10.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.http import Request -from scrapy.utils.defer import deferred_f_from_coro_f -from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase - -if TYPE_CHECKING: - from scrapy.core.downloader.handlers import DownloadHandlerProtocol - from tests.mockserver.http import MockServer - - -pytestmark = pytest.mark.requires_reactor # HTTP10DownloadHandler requires a reactor - - -class HTTP10DownloadHandlerMixin: - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): - """HTTP 1.0 test case""" - - def test_unsupported_scheme(self) -> None: # type: ignore[override] - pytest.skip("Check not implemented") - - @deferred_f_from_coro_f - async def test_protocol(self, mockserver: MockServer) -> None: - request = Request(mockserver.url("/host", is_secure=self.is_secure)) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.protocol == "HTTP/1.0" - - -class TestHttps10(TestHttp10): - is_secure = True - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase): - @deferred_f_from_coro_f - async def test_download_with_proxy_https_timeout(self): - pytest.skip("Not implemented") - - @deferred_f_from_coro_f - async def test_download_with_proxy_without_http_scheme(self): - pytest.skip("Not implemented") diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index f6e86d3dc..01fb26d85 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,9 +11,9 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from tests.test_downloader_handlers_http_base import ( - TestHttp11Base, + TestHttpBase, TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, @@ -41,11 +41,11 @@ def test_not_configured_without_reactor() -> None: HTTP11DownloadHandler.from_crawler(crawler) -class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): +class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): pass -class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base): +class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase): pass @@ -53,25 +53,25 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): pass -class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class TestHttps11InvalidDNSPattern( +class TestHttpsInvalidDNSPattern( HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class TestHttp11WithCrawler(TestHttpWithCrawlerBase): +class TestHttpWithCrawler(TestHttpWithCrawlerBase): @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -82,9 +82,9 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase): } -class TestHttps11WithCrawler(TestHttp11WithCrawler): +class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True -class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): +class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): pass diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 86fc071f8..3273a264e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -19,7 +19,7 @@ from scrapy.http import Request from scrapy.utils.defer import maybe_deferred_to_future from tests.test_downloader_handlers_http_base import ( TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, @@ -61,7 +61,7 @@ def test_not_configured_without_reactor() -> None: H2DownloadHandler.from_crawler(crawler) -class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): +class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): http2 = True handler_supports_http2_dataloss = False @@ -149,27 +149,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): await download_handler.download_request(request) -class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttp2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class TestHttps2InvalidDNSPattern( +class TestHttp2InvalidDNSPattern( H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass class TestHttp2WithCrawler(TestHttpWithCrawlerBase): - """HTTP 2.0 test case with MockServer""" - @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -194,7 +192,7 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase): pytest.skip("headers_received support is not implemented") -class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): +class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): is_secure = True expected_http_proxy_request_body = b"/" diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 66edf9325..2a781dace 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -55,9 +55,17 @@ if TYPE_CHECKING: class TestHttpBase(ABC): - is_secure = False + is_secure: bool = False + http2: bool = False # whether the handler supports per-request bindaddress - handler_supports_bindaddress_meta = True + handler_supports_bindaddress_meta: bool = True + # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a + # stream error (of type PROTOCOL_ERROR) so the client will send + # RST_STREAM. Some libraries do only this while e.g. h2 also closes the + # connection (see handling of ProtocolError in + # h2.connection.H2Connection.receive_data()), thus closing all streams that + # were using it, and we handle this as a normal exception. + handler_supports_http2_dataloss: bool = True # default headers added by the underlying library that cannot be suppressed always_present_req_headers: ClassVar[frozenset[str]] = frozenset() @@ -519,17 +527,6 @@ class TestHttpBase(ABC): else: assert latency > 0 - -class TestHttp11Base(TestHttpBase): - http2: bool = False - # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a - # stream error (of type PROTOCOL_ERROR) so the client will send - # RST_STREAM. Some libraries do only this while e.g. h2 also closes the - # connection (see handling of ProtocolError in - # h2.connection.H2Connection.receive_data()), thus closing all streams that - # were using it, and we handle this as a normal exception. - handler_supports_http2_dataloss: bool = True - @coroutine_test async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) @@ -803,7 +800,7 @@ class TestHttp11Base(TestHttpBase): ) -class TestHttps11Base(TestHttp11Base): +class TestHttpsBase(TestHttpBase): is_secure = True tls_log_message = ( diff --git a/tests/test_webclient.py b/tests/test_webclient.py deleted file mode 100644 index ac05d457d..000000000 --- a/tests/test_webclient.py +++ /dev/null @@ -1,404 +0,0 @@ -""" -Tests borrowed from the twisted.web.client tests. -""" - -from __future__ import annotations - -from urllib.parse import urlparse - -import OpenSSL.SSL -import pytest -from pytest_twisted import async_yield_fixture -from twisted.internet.defer import inlineCallbacks -from twisted.internet.testing import StringTransport -from twisted.protocols.policies import WrappingFactory -from twisted.web import resource, server, static, util -from twisted.web.client import _makeGetterFactory - -from scrapy.core.downloader import webclient as client -from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory -from scrapy.exceptions import DownloadTimeoutError -from scrapy.http import Headers, Request -from scrapy.utils.misc import build_from_crawler -from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import ( - ForeverTakingResource, - HostHeaderResource, - PayloadResource, -) -from tests.mockserver.utils import ssl_context_factory -from tests.test_core_downloader import TestContextFactoryBase - -# these tests are related to the Twisted HTTP code -pytestmark = pytest.mark.requires_reactor - - -def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): - """Adapted version of twisted.web.client.getPage""" - - def _clientfactory(url, *args, **kwargs): - url = to_unicode(url) - timeout = kwargs.pop("timeout", 0) - f = client.ScrapyHTTPClientFactory( - Request(url, *args, **kwargs), timeout=timeout - ) - f.deferred.addCallback(response_transform or (lambda r: r.body)) - return f - - return _makeGetterFactory( - to_bytes(url), - _clientfactory, - *args, - contextFactory=contextFactory, - **kwargs, - ).deferred - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestScrapyHTTPPageGetter: - def test_earlyHeaders(self): - # basic test stolen from twisted HTTPageGetter - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - body="some data", - headers={ - "Host": "example.net", - "User-Agent": "fooble", - "Cookie": "blah blah", - "Content-Length": "12981", - "Useful": "value", - }, - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Content-Length: 9\r\n" - b"Useful: value\r\n" - b"Connection: close\r\n" - b"User-Agent: fooble\r\n" - b"Host: example.net\r\n" - b"Cookie: blah blah\r\n" - b"\r\n" - b"some data", - ) - - # test minimal sent headers - factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar")) - self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n") - - # test a simple POST with body and content-type - factory = client.ScrapyHTTPClientFactory( - Request( - method="POST", - url="http://foo/bar", - body="name=value", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - ) - - self._test( - factory, - b"POST /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"Connection: close\r\n" - b"Content-Type: application/x-www-form-urlencoded\r\n" - b"Content-Length: 10\r\n" - b"\r\n" - b"name=value", - ) - - # test a POST method with no body provided - factory = client.ScrapyHTTPClientFactory( - Request(method="POST", url="http://foo/bar") - ) - - self._test( - factory, - b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n", - ) - - # test with single and multivalued headers - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - headers={ - "X-Meta-Single": "single", - "X-Meta-Multivalued": ["value1", "value2"], - }, - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"X-Meta-Multivalued: value1\r\n" - b"X-Meta-Multivalued: value2\r\n" - b"X-Meta-Single: single\r\n" - b"\r\n", - ) - - # same test with single and multivalued headers but using Headers class - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - headers=Headers( - { - "X-Meta-Single": "single", - "X-Meta-Multivalued": ["value1", "value2"], - } - ), - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"X-Meta-Multivalued: value1\r\n" - b"X-Meta-Multivalued: value2\r\n" - b"X-Meta-Single: single\r\n" - b"\r\n", - ) - - def _test(self, factory, testvalue): - transport = StringTransport() - protocol = client.ScrapyHTTPPageGetter() - protocol.factory = factory - protocol.makeConnection(transport) - assert set(transport.value().splitlines()) == set(testvalue.splitlines()) - return testvalue - - def test_non_standard_line_endings(self): - # regression test for: http://dev.scrapy.org/ticket/258 - factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar")) - protocol = client.ScrapyHTTPPageGetter() - protocol.factory = factory - protocol.headers = Headers() - protocol.dataReceived(b"HTTP/1.0 200 OK\n") - protocol.dataReceived(b"Hello: World\n") - protocol.dataReceived(b"Foo: Bar\n") - protocol.dataReceived(b"\n") - assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]}) - - -class EncodingResource(resource.Resource): - out_encoding = "cp1251" - - def render(self, request): - body = to_unicode(request.content.read()) - request.setHeader(b"content-encoding", self.out_encoding) - return body.encode(self.out_encoding) - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - # only sends 3 bytes even though it claims to send 5 - request.setHeader(b"content-length", b"5") - request.write(b"abc") - return b"" - - -class ErrorResource(resource.Resource): - def render(self, request): - request.setResponseCode(401) - if request.args.get(b"showlength"): - request.setHeader(b"content-length", b"0") - return b"" - - -class NoLengthResource(resource.Resource): - def render(self, request): - return b"nolength" - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestWebClient: - def _listen(self, site): - from twisted.internet import reactor - - return reactor.listenTCP(0, site, interface="127.0.0.1") - - @pytest.fixture - def wrapper(self, tmp_path): - (tmp_path / "file").write_bytes(b"0123456789") - r = static.File(str(tmp_path)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"error", ErrorResource()) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"encoding", EncodingResource()) - site = server.Site(r, timeout=None) - return WrappingFactory(site) - - @async_yield_fixture - async def server_port(self, wrapper): - port = self._listen(wrapper) - - yield port.getHost().port - - await port.stopListening() - - @pytest.fixture - def server_url(self, server_port): - return f"http://127.0.0.1:{server_port}/" - - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - body = yield getPage(server_url + "payload", body=s) - assert body == to_bytes(s) - - @inlineCallbacks - def testHostHeader(self, server_port, server_url): - # if we pass Host header explicitly, it should be used, otherwise - # it should extract from url - body = yield getPage(server_url + "host") - assert body == to_bytes(f"127.0.0.1:{server_port}") - body = yield getPage(server_url + "host", headers={"Host": "www.example.com"}) - assert body == to_bytes("www.example.com") - - @inlineCallbacks - def test_getPage(self, server_url): - """ - L{client.getPage} returns a L{Deferred} which is called back with - the body of the response if the default method B{GET} is used. - """ - body = yield getPage(server_url + "file") - assert body == b"0123456789" - - @inlineCallbacks - def test_getPageHead(self, server_url): - """ - L{client.getPage} returns a L{Deferred} which is called back with - the empty string if the method is C{HEAD} and there is a successful - response code. - """ - - def _getPage(method): - return getPage(server_url + "file", method=method) - - body = yield _getPage("head") - assert body == b"" - body = yield _getPage("HEAD") - assert body == b"" - - @inlineCallbacks - def test_timeoutNotTriggering(self, server_port, server_url): - """ - When a non-zero timeout is passed to L{getPage} and the page is - retrieved before the timeout period elapses, the L{Deferred} is - called back with the contents of the page. - """ - body = yield getPage(server_url + "host", timeout=100) - assert body == to_bytes(f"127.0.0.1:{server_port}") - - @inlineCallbacks - def test_timeoutTriggering(self, wrapper, server_url): - """ - When a non-zero timeout is passed to L{getPage} and that many - seconds elapse before the server responds to the request. the - L{Deferred} is errbacked with a L{DownloadTimeoutError}. - """ - with pytest.raises(DownloadTimeoutError): - yield getPage(server_url + "wait", timeout=0.000001) - # Clean up the server which is hanging around not doing - # anything. - connected = list(wrapper.protocols.keys()) - # There might be nothing here if the server managed to already see - # that the connection was lost. - if connected: - connected[0].transport.loseConnection() - - @inlineCallbacks - def testNotFound(self, server_url): - body = yield getPage(server_url + "notsuchfile") - assert b"404 - No Such Resource" in body - - @inlineCallbacks - def testFactoryInfo(self, server_url): - from twisted.internet import reactor - - url = server_url + "file" - parsed = urlparse(url) - factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(parsed.hostname, parsed.port, factory) - yield factory.deferred - assert factory.status == b"200" - assert factory.version.startswith(b"HTTP/") - assert factory.message == b"OK" - assert factory.response_headers[b"content-length"] == b"10" - - @inlineCallbacks - def testRedirect(self, server_url): - body = yield getPage(server_url + "redirect") - assert ( - body - == b'\n\n \n \n' - b' \n \n ' - b'click here\n \n\n' - ) - - @inlineCallbacks - def test_encoding(self, server_url): - """Test that non-standart body encoding matches - Content-Encoding header""" - original_body = b"\xd0\x81\xd1\x8e\xd0\xaf" - response = yield getPage( - server_url + "encoding", body=original_body, response_transform=lambda r: r - ) - content_encoding = to_unicode(response.headers[b"Content-Encoding"]) - assert content_encoding == EncodingResource.out_encoding - assert response.body.decode(content_encoding) == to_unicode(original_body) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestWebClientSSL(TestContextFactoryBase): - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - body = yield getPage(server_url + "payload", body=s) - assert body == to_bytes(s) - - -class TestWebClientCustomCiphersSSL(TestWebClientSSL): - # we try to use a cipher that is not enabled by default in OpenSSL - custom_ciphers = "CAMELLIA256-SHA" - context_factory = ssl_context_factory(cipher_string=custom_ciphers) - - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} - ) - client_context_factory = build_from_crawler( - _ScrapyClientContextFactory, crawler - ) - body = yield getPage( - server_url + "payload", body=s, contextFactory=client_context_factory - ) - assert body == to_bytes(s) - - @inlineCallbacks - def testPayloadDisabledCipher(self, server_url): - s = "0123456789" * 10 - crawler = get_crawler( - settings_dict={ - "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384" - } - ) - client_context_factory = build_from_crawler( - _ScrapyClientContextFactory, crawler - ) - with pytest.raises(OpenSSL.SSL.Error): - yield getPage( - server_url + "payload", body=s, contextFactory=client_context_factory - )