diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index dd2edebdd..98a74f8ce 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,13 +12,13 @@ Key takeaways: > Note: What follows is based on > https://raw.githubusercontent.com/jackyzha0/quartz/acfaa472253a432d350e9b6904c0cde14f8c487f/.github/pull_request_template.md -We more than welcome contributions, and are OK with the use of LLMs tools. How +We more than welcome contributions, and are OK with the use of LLM tools. How you use those tools depends on whether or not they make you more productive. But one thing that bugs us a lot are PRs that are made entirely with these tools, without any revision or any effort trying to refine their output whatsoever. This is just pure laziness, and unacceptable. Doing so will just -end up wasting everyone time (ours and yours). +end up wasting everyone's time (ours and yours). So to be the most productive for all parties, we would encourage any contributors to, at the very least, pay attention to what the model is doing, diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py index a8f2c6b68..e342e92cf 100644 --- a/docs/_ext/scrapyfixautodoc.py +++ b/docs/_ext/scrapyfixautodoc.py @@ -11,7 +11,7 @@ from sphinx.application import Sphinx def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: if not skip: - # autodocs was generating a text "alias of" for the following members + # autodoc was generating the text "alias of" for the following members return name in {"default_item_class", "default_selector_class"} return skip diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index e74afe49b..6d99c756e 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -46,12 +46,12 @@ the console you need to type:: Password: >>> -By default Username is ``scrapy`` and Password is autogenerated. The -autogenerated Password can be seen on Scrapy logs like the example below:: +By default, the username is ``scrapy`` and the password is autogenerated. The +autogenerated password can be seen on Scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 -Default Username and Password can be overridden by the settings +The default username and password can be overridden by the settings :setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. .. warning:: diff --git a/pyproject.toml b/pyproject.toml index ea7893e2a..2132c5c50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,18 +122,6 @@ implicit_reexport = true module = "scrapy.settings.default_settings" ignore_errors = true -# deprecated modules -[[tool.mypy.overrides]] -module = [ - "scrapy.core.downloader.webclient", - "tests.test_webclient", -] -allow_any_generics = true -allow_untyped_calls = true -allow_untyped_defs = true -check_untyped_defs = false -warn_return_any = false - # usually no type hints [[tool.mypy.overrides]] module = [ diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 19fc77959..bbda7efcd 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -114,10 +114,10 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): acceptableCiphers=self.tls_ciphers, ) - # kept for old-style HTTP/1.0 downloader context twisted calls, - # e.g. connectSSL() # should be removed together with ScrapyClientContextFactory - def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: + def getContext( + self, hostname: Any = None, port: Any = None + ) -> SSL.Context: # pragma: no cover return self._get_context() def _get_context(self) -> SSL.Context: diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index a7b592d00..dc1b1e375 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,7 +1,6 @@ # pragma: no file cover import warnings -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) @@ -16,6 +15,5 @@ warnings.warn( ) __all__ = [ - "HTTP10DownloadHandler", "HTTPDownloadHandler", ] diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py deleted file mode 100644 index f5d1bbd76..000000000 --- a/scrapy/core/downloader/handlers/http10.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Download handlers for http and https schemes""" - -from __future__ import annotations - -import warnings -from typing import TYPE_CHECKING - -from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import to_unicode - -if TYPE_CHECKING: - from twisted.internet.interfaces import IConnector - - # typing.Self requires Python 3.11 - from typing_extensions import Self - - from scrapy import Request - from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory - from scrapy.crawler import Crawler - from scrapy.http import Response - from scrapy.settings import BaseSettings - - -class HTTP10DownloadHandler: - lazy = False - - def __init__(self, settings: BaseSettings, crawler: Crawler): - warnings.warn( - "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): # pragma: no cover - raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") - self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( - settings["DOWNLOADER_HTTPCLIENTFACTORY"] - ) - if settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": - self.ClientContextFactory: type[_ScrapyClientContextFactory] = ( - _ScrapyClientContextFactory - ) - else: # pragma: no cover - warnings.warn( - "The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.ClientContextFactory = load_object( - settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] - ) - self._settings: BaseSettings = settings - self._crawler: Crawler = crawler - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings, crawler) - - async def download_request(self, request: Request) -> Response: - factory = self.HTTPClientFactory(request) - self._connect(factory) - return await maybe_deferred_to_future(factory.deferred) - - def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: - from twisted.internet import reactor - - host, port = to_unicode(factory.host), factory.port - if factory.scheme == b"https": - client_context_factory = build_from_crawler( - self.ClientContextFactory, - self._crawler, - ) - return reactor.connectSSL(host, port, factory, client_context_factory) - return reactor.connectTCP(host, port, factory) - - async def close(self) -> None: - pass diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py deleted file mode 100644 index 2a550cf78..000000000 --- a/scrapy/core/downloader/webclient.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler.""" - -from __future__ import annotations - -import warnings -from time import monotonic, time -from typing import TYPE_CHECKING -from urllib.parse import urldefrag, urlparse, urlunparse - -from twisted.internet import defer -from twisted.internet.protocol import ClientFactory -from twisted.web.http import HTTPClient - -from scrapy.exceptions import DownloadTimeoutError, ScrapyDeprecationWarning -from scrapy.http import Headers, Response -from scrapy.responsetypes import responsetypes -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode - -if TYPE_CHECKING: - from scrapy import Request - - -class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b"\n" - - def __init__(self): - warnings.warn( - "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__() - - def connectionMade(self): - self.headers = Headers() # bucket for response headers - - # Method command - self.sendCommand(self.factory.method, self.factory.path) - # Headers - for key, values in self.factory.headers.items(): - for value in values: - self.sendHeader(key, value) - self.endHeaders() - # Body - if self.factory.body is not None: - self.transport.write(self.factory.body) - - def lineReceived(self, line): - return HTTPClient.lineReceived(self, line.rstrip()) - - def handleHeader(self, key, value): - self.headers.appendlist(key, value) - - def handleStatus(self, version, status, message): - self.factory.gotStatus(version, status, message) - - def handleEndHeaders(self): - self.factory.gotHeaders(self.headers) - - def connectionLost(self, reason): - self._connection_lost_reason = reason - HTTPClient.connectionLost(self, reason) - self.factory.noPage(reason) - - def handleResponse(self, response): - if self.factory.method.upper() == b"HEAD": - self.factory.page(b"") - elif self.length is not None and self.length > 0: - self.factory.noPage(self._connection_lost_reason) - else: - self.factory.page(response) - self.transport.loseConnection() - - def timeout(self): - self.transport.loseConnection() - - # transport cleanup needed for HTTPS connections - if self.factory.url.startswith(b"https"): - self.transport.stopProducing() - - self.factory.noPage( - DownloadTimeoutError( - f"Getting {self.factory.url} took longer " - f"than {self.factory.timeout} seconds." - ) - ) - - -# This class used to inherit from Twisted’s -# twisted.web.client.HTTPClientFactory. When that class was deprecated in -# Twisted (https://github.com/twisted/twisted/pull/643), we merged its -# non-overridden code into this class. -class ScrapyHTTPClientFactory(ClientFactory): - protocol = ScrapyHTTPPageGetter - - waiting = 1 - noisy = False - followRedirect = False - afterFoundGet = False - - def _build_response(self, body, request): - request.meta["download_latency"] = ( - self._headers_time_mono - self._start_time_mono - ) - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) - return respcls( - url=self._url, - status=status, - headers=headers, - body=body, - protocol=to_unicode(self.version), - ) - - def _set_connection_attributes(self, request): - proxy = request.meta.get("proxy") - if proxy: - proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii")) - self.scheme = proxy_parsed.scheme - self.host = proxy_parsed.hostname - self.port = proxy_parsed.port - self.netloc = proxy_parsed.netloc - if self.port is None: - self.port = 443 if proxy_parsed.scheme == b"https" else 80 - self.path = self.url - else: - parsed = urlparse_cached(request) - path_str = urlunparse( - ("", "", parsed.path or "/", parsed.params, parsed.query, "") - ) - self.path = to_bytes(path_str, encoding="ascii") - assert parsed.hostname is not None - self.host = to_bytes(parsed.hostname, encoding="ascii") - self.port = parsed.port - self.scheme = to_bytes(parsed.scheme, encoding="ascii") - self.netloc = to_bytes(parsed.netloc, encoding="ascii") - if self.port is None: - self.port = 443 if self.scheme == b"https" else 80 - - def __init__(self, request: Request, timeout: float = 180): - warnings.warn( - "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - self._url: str = urldefrag(request.url)[0] - # converting to bytes to comply to Twisted interface - self.url: bytes = to_bytes(self._url, encoding="ascii") - self.method: bytes = to_bytes(request.method, encoding="ascii") - self.body: bytes | None = request.body or None - self.headers: Headers = Headers(request.headers) - self.response_headers: Headers | None = None - self.timeout: float = request.meta.get("download_timeout") or timeout - self.start_time: float = time() - self._start_time_mono: float = monotonic() - self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( - self._build_response, request - ) - - # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected - # to have _disconnectedDeferred. See Twisted r32329. - # As Scrapy implements it's own logic to handle redirects is not - # needed to add the callback _waitForDisconnect. - # Specifically this avoids the AttributeError exception when - # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred() - - self._set_connection_attributes(request) - - # set Host header based on url - self.headers.setdefault("Host", self.netloc) - - # set Content-Length based len of body - if self.body is not None: - self.headers["Content-Length"] = len(self.body) - # just in case a broken http/1.1 decides to keep connection alive - self.headers.setdefault("Connection", "close") - # Content-Length must be specified in POST method even with no body - elif self.method == b"POST": - self.headers["Content-Length"] = 0 - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}: {self._url}>" - - def _cancelTimeout(self, result, timeoutCall): - if timeoutCall.active(): - timeoutCall.cancel() - return result - - def buildProtocol(self, addr): - p = ClientFactory.buildProtocol(self, addr) - p.followRedirect = self.followRedirect - p.afterFoundGet = self.afterFoundGet - if self.timeout: - from twisted.internet import reactor - - timeoutCall = reactor.callLater(self.timeout, p.timeout) - self.deferred.addBoth(self._cancelTimeout, timeoutCall) - return p - - def gotHeaders(self, headers): - self.headers_time = time() - self._headers_time_mono = monotonic() - self.response_headers = headers - - def gotStatus(self, version, status, message): - """ - Set the status of the request on us. - @param version: The HTTP version. - @type version: L{bytes} - @param status: The HTTP status code, an integer represented as a - bytestring. - @type status: L{bytes} - @param message: The HTTP status message. - @type message: L{bytes} - """ - self.version, self.status, self.message = version, status, message - - def page(self, page): - if self.waiting: - self.waiting = 0 - self.deferred.callback(page) - - def noPage(self, reason): - if self.waiting: - self.waiting = 0 - self.deferred.errback(reason) - - def clientConnectionFailed(self, _, reason): - """ - When a connection attempt fails, the request cannot be issued. If no - result has yet been provided to the result Deferred, provide the - connection failure reason as an error result. - """ - if self.waiting: - self.waiting = 0 - # If the connection attempt failed, there is nothing more to - # disconnect, so just fire that Deferred now. - self._disconnectedDeferred.callback(None) - self.deferred.errback(reason) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7b54a9c14..5d671feb3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -54,7 +54,6 @@ __all__ = [ "DOWNLOADER_CLIENT_TLS_CIPHERS", "DOWNLOADER_CLIENT_TLS_METHOD", "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING", - "DOWNLOADER_HTTPCLIENTFACTORY", "DOWNLOADER_MIDDLEWARES", "DOWNLOADER_MIDDLEWARES_BASE", "DOWNLOADER_STATS", @@ -275,10 +274,6 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False -DOWNLOADER_HTTPCLIENTFACTORY = ( - "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" -) - DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 3b1796af7..a0e3fccb3 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -9,9 +9,9 @@ import pytest from scrapy import Request from tests.test_downloader_handlers_http_base import ( - TestHttp11Base, + TestHttpBase, TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, @@ -42,7 +42,7 @@ class HttpxDownloadHandlerMixin: return HttpxDownloadHandler -class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): +class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): handler_supports_bindaddress_meta = False @pytest.mark.skipif( @@ -77,7 +77,7 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): ) -class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base): +class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): handler_supports_bindaddress_meta = False tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" @@ -90,25 +90,25 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): pass -class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class TestHttps11InvalidDNSPattern( +class TestHttpsInvalidDNSPattern( HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class TestHttp11WithCrawler(TestHttpWithCrawlerBase): +class TestHttpWithCrawler(TestHttpWithCrawlerBase): @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -119,7 +119,7 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase): } -class TestHttps11WithCrawler(TestHttp11WithCrawler): +class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True @pytest.mark.skip(reason="response.certificate is not implemented") @@ -129,10 +129,10 @@ class TestHttps11WithCrawler(TestHttp11WithCrawler): @pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): pass @pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): is_secure = True diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py deleted file mode 100644 index 8281337b0..000000000 --- a/tests/test_downloader_handler_twisted_http10.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.http import Request -from scrapy.utils.defer import deferred_f_from_coro_f -from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase - -if TYPE_CHECKING: - from scrapy.core.downloader.handlers import DownloadHandlerProtocol - from tests.mockserver.http import MockServer - - -pytestmark = pytest.mark.requires_reactor # HTTP10DownloadHandler requires a reactor - - -class HTTP10DownloadHandlerMixin: - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): - """HTTP 1.0 test case""" - - def test_unsupported_scheme(self) -> None: # type: ignore[override] - pytest.skip("Check not implemented") - - @deferred_f_from_coro_f - async def test_protocol(self, mockserver: MockServer) -> None: - request = Request(mockserver.url("/host", is_secure=self.is_secure)) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.protocol == "HTTP/1.0" - - -class TestHttps10(TestHttp10): - is_secure = True - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase): - @deferred_f_from_coro_f - async def test_download_with_proxy_https_timeout(self): - pytest.skip("Not implemented") - - @deferred_f_from_coro_f - async def test_download_with_proxy_without_http_scheme(self): - pytest.skip("Not implemented") diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index f6e86d3dc..01fb26d85 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,9 +11,9 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from tests.test_downloader_handlers_http_base import ( - TestHttp11Base, + TestHttpBase, TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, @@ -41,11 +41,11 @@ def test_not_configured_without_reactor() -> None: HTTP11DownloadHandler.from_crawler(crawler) -class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): +class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): pass -class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base): +class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase): pass @@ -53,25 +53,25 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): pass -class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class TestHttps11InvalidDNSPattern( +class TestHttpsInvalidDNSPattern( HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class TestHttp11WithCrawler(TestHttpWithCrawlerBase): +class TestHttpWithCrawler(TestHttpWithCrawlerBase): @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -82,9 +82,9 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase): } -class TestHttps11WithCrawler(TestHttp11WithCrawler): +class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True -class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): +class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): pass diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 86fc071f8..3273a264e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -19,7 +19,7 @@ from scrapy.http import Request from scrapy.utils.defer import maybe_deferred_to_future from tests.test_downloader_handlers_http_base import ( TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, @@ -61,7 +61,7 @@ def test_not_configured_without_reactor() -> None: H2DownloadHandler.from_crawler(crawler) -class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): +class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): http2 = True handler_supports_http2_dataloss = False @@ -149,27 +149,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): await download_handler.download_request(request) -class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttp2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class TestHttps2InvalidDNSPattern( +class TestHttp2InvalidDNSPattern( H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass class TestHttp2WithCrawler(TestHttpWithCrawlerBase): - """HTTP 2.0 test case with MockServer""" - @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -194,7 +192,7 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase): pytest.skip("headers_received support is not implemented") -class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): +class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): is_secure = True expected_http_proxy_request_body = b"/" diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 66edf9325..2a781dace 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -55,9 +55,17 @@ if TYPE_CHECKING: class TestHttpBase(ABC): - is_secure = False + is_secure: bool = False + http2: bool = False # whether the handler supports per-request bindaddress - handler_supports_bindaddress_meta = True + handler_supports_bindaddress_meta: bool = True + # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a + # stream error (of type PROTOCOL_ERROR) so the client will send + # RST_STREAM. Some libraries do only this while e.g. h2 also closes the + # connection (see handling of ProtocolError in + # h2.connection.H2Connection.receive_data()), thus closing all streams that + # were using it, and we handle this as a normal exception. + handler_supports_http2_dataloss: bool = True # default headers added by the underlying library that cannot be suppressed always_present_req_headers: ClassVar[frozenset[str]] = frozenset() @@ -519,17 +527,6 @@ class TestHttpBase(ABC): else: assert latency > 0 - -class TestHttp11Base(TestHttpBase): - http2: bool = False - # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a - # stream error (of type PROTOCOL_ERROR) so the client will send - # RST_STREAM. Some libraries do only this while e.g. h2 also closes the - # connection (see handling of ProtocolError in - # h2.connection.H2Connection.receive_data()), thus closing all streams that - # were using it, and we handle this as a normal exception. - handler_supports_http2_dataloss: bool = True - @coroutine_test async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) @@ -803,7 +800,7 @@ class TestHttp11Base(TestHttpBase): ) -class TestHttps11Base(TestHttp11Base): +class TestHttpsBase(TestHttpBase): is_secure = True tls_log_message = ( diff --git a/tests/test_webclient.py b/tests/test_webclient.py deleted file mode 100644 index ac05d457d..000000000 --- a/tests/test_webclient.py +++ /dev/null @@ -1,404 +0,0 @@ -""" -Tests borrowed from the twisted.web.client tests. -""" - -from __future__ import annotations - -from urllib.parse import urlparse - -import OpenSSL.SSL -import pytest -from pytest_twisted import async_yield_fixture -from twisted.internet.defer import inlineCallbacks -from twisted.internet.testing import StringTransport -from twisted.protocols.policies import WrappingFactory -from twisted.web import resource, server, static, util -from twisted.web.client import _makeGetterFactory - -from scrapy.core.downloader import webclient as client -from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory -from scrapy.exceptions import DownloadTimeoutError -from scrapy.http import Headers, Request -from scrapy.utils.misc import build_from_crawler -from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import ( - ForeverTakingResource, - HostHeaderResource, - PayloadResource, -) -from tests.mockserver.utils import ssl_context_factory -from tests.test_core_downloader import TestContextFactoryBase - -# these tests are related to the Twisted HTTP code -pytestmark = pytest.mark.requires_reactor - - -def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): - """Adapted version of twisted.web.client.getPage""" - - def _clientfactory(url, *args, **kwargs): - url = to_unicode(url) - timeout = kwargs.pop("timeout", 0) - f = client.ScrapyHTTPClientFactory( - Request(url, *args, **kwargs), timeout=timeout - ) - f.deferred.addCallback(response_transform or (lambda r: r.body)) - return f - - return _makeGetterFactory( - to_bytes(url), - _clientfactory, - *args, - contextFactory=contextFactory, - **kwargs, - ).deferred - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestScrapyHTTPPageGetter: - def test_earlyHeaders(self): - # basic test stolen from twisted HTTPageGetter - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - body="some data", - headers={ - "Host": "example.net", - "User-Agent": "fooble", - "Cookie": "blah blah", - "Content-Length": "12981", - "Useful": "value", - }, - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Content-Length: 9\r\n" - b"Useful: value\r\n" - b"Connection: close\r\n" - b"User-Agent: fooble\r\n" - b"Host: example.net\r\n" - b"Cookie: blah blah\r\n" - b"\r\n" - b"some data", - ) - - # test minimal sent headers - factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar")) - self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n") - - # test a simple POST with body and content-type - factory = client.ScrapyHTTPClientFactory( - Request( - method="POST", - url="http://foo/bar", - body="name=value", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - ) - - self._test( - factory, - b"POST /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"Connection: close\r\n" - b"Content-Type: application/x-www-form-urlencoded\r\n" - b"Content-Length: 10\r\n" - b"\r\n" - b"name=value", - ) - - # test a POST method with no body provided - factory = client.ScrapyHTTPClientFactory( - Request(method="POST", url="http://foo/bar") - ) - - self._test( - factory, - b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n", - ) - - # test with single and multivalued headers - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - headers={ - "X-Meta-Single": "single", - "X-Meta-Multivalued": ["value1", "value2"], - }, - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"X-Meta-Multivalued: value1\r\n" - b"X-Meta-Multivalued: value2\r\n" - b"X-Meta-Single: single\r\n" - b"\r\n", - ) - - # same test with single and multivalued headers but using Headers class - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - headers=Headers( - { - "X-Meta-Single": "single", - "X-Meta-Multivalued": ["value1", "value2"], - } - ), - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"X-Meta-Multivalued: value1\r\n" - b"X-Meta-Multivalued: value2\r\n" - b"X-Meta-Single: single\r\n" - b"\r\n", - ) - - def _test(self, factory, testvalue): - transport = StringTransport() - protocol = client.ScrapyHTTPPageGetter() - protocol.factory = factory - protocol.makeConnection(transport) - assert set(transport.value().splitlines()) == set(testvalue.splitlines()) - return testvalue - - def test_non_standard_line_endings(self): - # regression test for: http://dev.scrapy.org/ticket/258 - factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar")) - protocol = client.ScrapyHTTPPageGetter() - protocol.factory = factory - protocol.headers = Headers() - protocol.dataReceived(b"HTTP/1.0 200 OK\n") - protocol.dataReceived(b"Hello: World\n") - protocol.dataReceived(b"Foo: Bar\n") - protocol.dataReceived(b"\n") - assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]}) - - -class EncodingResource(resource.Resource): - out_encoding = "cp1251" - - def render(self, request): - body = to_unicode(request.content.read()) - request.setHeader(b"content-encoding", self.out_encoding) - return body.encode(self.out_encoding) - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - # only sends 3 bytes even though it claims to send 5 - request.setHeader(b"content-length", b"5") - request.write(b"abc") - return b"" - - -class ErrorResource(resource.Resource): - def render(self, request): - request.setResponseCode(401) - if request.args.get(b"showlength"): - request.setHeader(b"content-length", b"0") - return b"" - - -class NoLengthResource(resource.Resource): - def render(self, request): - return b"nolength" - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestWebClient: - def _listen(self, site): - from twisted.internet import reactor - - return reactor.listenTCP(0, site, interface="127.0.0.1") - - @pytest.fixture - def wrapper(self, tmp_path): - (tmp_path / "file").write_bytes(b"0123456789") - r = static.File(str(tmp_path)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"error", ErrorResource()) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"encoding", EncodingResource()) - site = server.Site(r, timeout=None) - return WrappingFactory(site) - - @async_yield_fixture - async def server_port(self, wrapper): - port = self._listen(wrapper) - - yield port.getHost().port - - await port.stopListening() - - @pytest.fixture - def server_url(self, server_port): - return f"http://127.0.0.1:{server_port}/" - - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - body = yield getPage(server_url + "payload", body=s) - assert body == to_bytes(s) - - @inlineCallbacks - def testHostHeader(self, server_port, server_url): - # if we pass Host header explicitly, it should be used, otherwise - # it should extract from url - body = yield getPage(server_url + "host") - assert body == to_bytes(f"127.0.0.1:{server_port}") - body = yield getPage(server_url + "host", headers={"Host": "www.example.com"}) - assert body == to_bytes("www.example.com") - - @inlineCallbacks - def test_getPage(self, server_url): - """ - L{client.getPage} returns a L{Deferred} which is called back with - the body of the response if the default method B{GET} is used. - """ - body = yield getPage(server_url + "file") - assert body == b"0123456789" - - @inlineCallbacks - def test_getPageHead(self, server_url): - """ - L{client.getPage} returns a L{Deferred} which is called back with - the empty string if the method is C{HEAD} and there is a successful - response code. - """ - - def _getPage(method): - return getPage(server_url + "file", method=method) - - body = yield _getPage("head") - assert body == b"" - body = yield _getPage("HEAD") - assert body == b"" - - @inlineCallbacks - def test_timeoutNotTriggering(self, server_port, server_url): - """ - When a non-zero timeout is passed to L{getPage} and the page is - retrieved before the timeout period elapses, the L{Deferred} is - called back with the contents of the page. - """ - body = yield getPage(server_url + "host", timeout=100) - assert body == to_bytes(f"127.0.0.1:{server_port}") - - @inlineCallbacks - def test_timeoutTriggering(self, wrapper, server_url): - """ - When a non-zero timeout is passed to L{getPage} and that many - seconds elapse before the server responds to the request. the - L{Deferred} is errbacked with a L{DownloadTimeoutError}. - """ - with pytest.raises(DownloadTimeoutError): - yield getPage(server_url + "wait", timeout=0.000001) - # Clean up the server which is hanging around not doing - # anything. - connected = list(wrapper.protocols.keys()) - # There might be nothing here if the server managed to already see - # that the connection was lost. - if connected: - connected[0].transport.loseConnection() - - @inlineCallbacks - def testNotFound(self, server_url): - body = yield getPage(server_url + "notsuchfile") - assert b"404 - No Such Resource" in body - - @inlineCallbacks - def testFactoryInfo(self, server_url): - from twisted.internet import reactor - - url = server_url + "file" - parsed = urlparse(url) - factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(parsed.hostname, parsed.port, factory) - yield factory.deferred - assert factory.status == b"200" - assert factory.version.startswith(b"HTTP/") - assert factory.message == b"OK" - assert factory.response_headers[b"content-length"] == b"10" - - @inlineCallbacks - def testRedirect(self, server_url): - body = yield getPage(server_url + "redirect") - assert ( - body - == b'\n\n
\n \n' - b' \n \n ' - b'click here\n \n\n' - ) - - @inlineCallbacks - def test_encoding(self, server_url): - """Test that non-standart body encoding matches - Content-Encoding header""" - original_body = b"\xd0\x81\xd1\x8e\xd0\xaf" - response = yield getPage( - server_url + "encoding", body=original_body, response_transform=lambda r: r - ) - content_encoding = to_unicode(response.headers[b"Content-Encoding"]) - assert content_encoding == EncodingResource.out_encoding - assert response.body.decode(content_encoding) == to_unicode(original_body) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestWebClientSSL(TestContextFactoryBase): - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - body = yield getPage(server_url + "payload", body=s) - assert body == to_bytes(s) - - -class TestWebClientCustomCiphersSSL(TestWebClientSSL): - # we try to use a cipher that is not enabled by default in OpenSSL - custom_ciphers = "CAMELLIA256-SHA" - context_factory = ssl_context_factory(cipher_string=custom_ciphers) - - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} - ) - client_context_factory = build_from_crawler( - _ScrapyClientContextFactory, crawler - ) - body = yield getPage( - server_url + "payload", body=s, contextFactory=client_context_factory - ) - assert body == to_bytes(s) - - @inlineCallbacks - def testPayloadDisabledCipher(self, server_url): - s = "0123456789" * 10 - crawler = get_crawler( - settings_dict={ - "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384" - } - ) - client_context_factory = build_from_crawler( - _ScrapyClientContextFactory, crawler - ) - with pytest.raises(OpenSSL.SSL.Error): - yield getPage( - server_url + "payload", body=s, contextFactory=client_context_factory - )