diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py new file mode 100644 index 000000000..807c8c4cb --- /dev/null +++ b/tests/test_downloader_handler_twisted_http10.py @@ -0,0 +1,46 @@ +"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.http import Request +from scrapy.spiders import Spider +from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class HTTP10DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): + """HTTP 1.0 test case""" + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + + +class TestHttps10(TestHttp10): + scheme = "https" + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase): + def test_download_with_proxy_https_timeout(self): + pytest.skip("Not implemented") + + def test_download_with_proxy_without_http_scheme(self): + pytest.skip("Not implemented") diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py new file mode 100644 index 000000000..70f55e787 --- /dev/null +++ b/tests/test_downloader_handler_twisted_http11.py @@ -0,0 +1,69 @@ +"""Tests for scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from tests.test_downloader_handlers_http_base import ( + TestHttp11Base, + TestHttpMockServerBase, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, + TestSimpleHttpsBase, +) + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class HTTP11DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler + + +class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): + pass + + +class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base): + pass + + +class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class Https11WrongHostnameTestCase( + HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase +): + pass + + +class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class Https11InvalidDNSPattern( + HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttp11MockServer(TestHttpMockServerBase): + @property + def settings_dict(self) -> dict[str, Any] | None: + return None # default handler settings + + +class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + pass diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handler_twisted_http2.py similarity index 73% rename from tests/test_downloader_handlers_http2.py rename to tests/test_downloader_handler_twisted_http2.py index c74c09cbb..46322a747 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -1,4 +1,9 @@ +"""Tests for scrapy.core.downloader.handlers.http2.H2DownloadHandler.""" + +from __future__ import annotations + import json +from typing import TYPE_CHECKING, Any from unittest import mock import pytest @@ -8,60 +13,42 @@ from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED -from scrapy.core.downloader.handlers import DownloadHandlerProtocol from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory -from tests.test_downloader_handlers import ( +from tests.test_downloader_handlers_http_base import ( + TestHttpMockServerBase, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, UriResource, ) +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + pytestmark = pytest.mark.skipif( not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" ) -class BaseTestClasses: - # A hack to prevent tests from the imported classes to run here too. - # See https://stackoverflow.com/q/1323455/113586 for other ways. - from tests.test_downloader_handlers import ( - TestHttp11MockServer as TestHttp11MockServer, - ) - from tests.test_downloader_handlers import ( - TestHttp11Proxy as TestHttp11Proxy, - ) - from tests.test_downloader_handlers import ( - TestHttps11 as TestHttps11, - ) - from tests.test_downloader_handlers import ( - TestHttps11CustomCiphers as TestHttps11CustomCiphers, - ) - from tests.test_downloader_handlers import ( - TestHttps11InvalidDNSId as TestHttps11InvalidDNSId, - ) - from tests.test_downloader_handlers import ( - TestHttps11InvalidDNSPattern as TestHttps11InvalidDNSPattern, - ) - from tests.test_downloader_handlers import ( - TestHttps11WrongHostname as TestHttps11WrongHostname, - ) - - -def _get_dh() -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - return H2DownloadHandler - - -class TestHttps2(BaseTestClasses.TestHttps11): - scheme = "https" - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - +class H2DownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() + # the import can fail when H2_ENABLED is False + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + + return H2DownloadHandler + + +class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -179,42 +166,37 @@ class TestHttps2(BaseTestClasses.TestHttps11): return d -class Https2WrongHostnameTestCase(BaseTestClasses.TestHttps11WrongHostname): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass -class Https2InvalidDNSId(BaseTestClasses.TestHttps11InvalidDNSId): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass -class Https2InvalidDNSPattern(BaseTestClasses.TestHttps11InvalidDNSPattern): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase): + pass -class Https2CustomCiphers(BaseTestClasses.TestHttps11CustomCiphers): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass -class Http2MockServerTestCase(BaseTestClasses.TestHttp11MockServer): +class Http2MockServerTestCase(TestHttpMockServerBase): """HTTP 2.0 test case with MockServer""" - settings_dict = { - "DOWNLOAD_HANDLERS": { - "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" + } } - } + is_secure = True -class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): +class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -224,10 +206,6 @@ class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): expected_http_proxy_request_body = b"/" - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() - def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bc18e76e1..fc6ac5aee 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,52 +1,35 @@ +"""Tests for DownloadHandlers and for specific non-HTTP download handlers.""" + from __future__ import annotations import contextlib import os import shutil import sys -from abc import ABC, abstractmethod from pathlib import Path from tempfile import mkdtemp, mkstemp -from unittest import SkipTest, mock +from unittest import mock import pytest -from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal -from twisted.internet import defer, error, reactor +from twisted.internet import reactor from twisted.protocols.ftp import FTPFactory, FTPRealm -from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest -from twisted.web import resource, server, static, util -from twisted.web.client import ResponseFailed -from twisted.web.http import _DataLoss from w3lib.url import path_to_file_uri -from scrapy.core.downloader.handlers import DownloadHandlerProtocol, DownloadHandlers +from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.http import Headers, HtmlResponse, Request +from scrapy.http import HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver import ( - Echo, - ForeverTakingResource, - HostHeaderResource, - MockServer, - NoLengthResource, - PayloadResource, - ssl_context_factory, -) -from tests.spiders import SingleRequestSpider class DummyDH: @@ -137,696 +120,6 @@ class TestFile(unittest.TestCase): return self.assertFailure(d, OSError) -class ContentLengthHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of the Content-Length - header from the request. - """ - - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] - - -class ChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - # Disable terminating chunk on finish. - request.chunked = False - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - def response(): - request.setHeader(b"Content-Length", b"20") - request.write(b"partial") - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -def closeConnection(request): - # We have to force a disconnection for HTTP/1.1 clients. Otherwise - # client keeps the connection open waiting for more data. - request.channel.loseConnection() - request.finish() - - -class EmptyContentTypeHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of request body - without content-type header in response. - """ - - def render(self, request): - request.setHeader("content-type", "") - return request.content.read() - - -class LargeChunkedFileResource(resource.Resource): - def render(self, request): - def response(): - for i in range(1024): - request.write(b"x" * 1024) - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class DuplicateHeaderResource(resource.Resource): - def render(self, request): - request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) - return b"" - - -class TestHttp(unittest.TestCase, ABC): - scheme = "http" - - # only used for HTTPS tests - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - - @property - @abstractmethod - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - raise NotImplementedError - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"chunked", ChunkedResource()) - r.putChild(b"broken-chunked", BrokenChunkedResource()) - r.putChild(b"contentlength", ContentLengthHeaderResource()) - r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - r.putChild(b"largechunkedfile", LargeChunkedFileResource()) - r.putChild(b"duplicate-header", DuplicateHeaderResource()) - r.putChild(b"echo", Echo()) - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = "localhost" - if self.scheme == "https": - # Using WrappingFactory do not enable HTTP/2 failing all the - # tests with H2DownloadHandler - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host, - ) - else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_head(self): - request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d - - def test_redirect_status(self): - request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - def test_redirect_status_head(self): - request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest != "default" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - - # client connects but no data is received - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest != "default" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - # client connects, server send headers and some body bytes but hangs - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - assert response.body == to_bytes(f"{self.host}:{self.portno}") - assert not request.headers - - request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_host_header_seted_in_request_headers(self): - host = self.host + ":" + str(self.portno) - - def _test(response): - assert response.body == host.encode() - assert request.headers.get("Host") == host.encode() - - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_request_headers(self): - """Tests if "Content-Length: 0" is sent for bodyless POST requests. - - This is not strictly required by HTTP RFCs but can cause trouble - for some web servers. - See: - https://github.com/scrapy/scrapy/issues/823 - https://issues.apache.org/jira/browse/TS-2902 - https://github.com/kennethreitz/requests/issues/405 - https://bugs.python.org/issue14721 - """ - - def _test(response): - assert response.body == b"0" - - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - assert len(contentlengths) == 1 - assert contentlengths == [b"0"] - - request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_payload(self): - body = b"1" * 100 # PayloadResource requires body length to be 100 - request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d - - def test_response_header_content_length(self): - request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - - request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) - - def test_response_class_from_body(self): - return self._test_response_class( - "foo", - b"\n.", - HtmlResponse, - ) - - def test_get_duplicate_header(self): - def _test(response): - assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - - request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10(TestHttp): - """HTTP 1.0 test case""" - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d - - -class TestHttps10(TestHttp10): - scheme = "https" - - -class TestHttp11(TestHttp): - """HTTP 1.1 test case""" - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - def test_download_without_maxsize_limit(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_response_class_choosing_request(self): - """Tests choosing of correct response type - in case of Content-Type is empty but body contains text. - """ - body = b"Some plain text\ndata with tabs\t and null bytes\0" - - def _test_type(response): - assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - - request = Request(self.getURL("nocontenttype"), body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(_test_type) - return d - - @defer.inlineCallbacks - def test_download_with_maxsize(self): - request = Request(self.getURL("file")) - - # 10 is minimal size for this request and the limit is only counted on - # response body. (regardless of headers) - d = self.download_request(request, Spider("foo", download_maxsize=10)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d - - d = self.download_request(request, Spider("foo", download_maxsize=9)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: - request = Request(self.getURL("largechunkedfile")) - - def check(logger): - logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(0.1, d.callback, logger) - yield d - - @defer.inlineCallbacks - def test_download_with_maxsize_per_req(self): - meta = {"download_maxsize": 2} - request = Request(self.getURL("file"), meta=meta) - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - @defer.inlineCallbacks - def test_download_with_small_maxsize_per_spider(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=2)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - def test_download_with_large_maxsize_per_spider(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=100)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_chunked_content(self): - request = Request(self.getURL("chunked")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"chunked content\n") - return d - - def test_download_broken_content_cause_data_loss(self, url="broken"): - request = Request(self.getURL(url)) - d = self.download_request(request, Spider("foo")) - - def checkDataLoss(failure): - if failure.check(ResponseFailed) and any( - r.check(_DataLoss) for r in failure.value.reasons - ): - return None - return failure - - d.addCallback(lambda _: self.fail("No DataLoss exception")) - d.addErrback(checkDataLoss) - return d - - def test_download_broken_chunked_content_cause_data_loss(self): - return self.test_download_broken_content_cause_data_loss("broken-chunked") - - def test_download_broken_content_allow_data_loss(self, url="broken"): - request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d - - def test_download_broken_chunked_content_allow_data_loss(self): - return self.test_download_broken_content_allow_data_loss("broken-chunked") - - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): - crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) - download_handler = build_from_crawler(self.download_handler_cls, crawler) - request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - return self.test_download_broken_content_allow_data_loss_via_setting( - "broken-chunked" - ) - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.1") - return d - - -class TestHttps11(TestHttp11): - scheme = "https" - - tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' - ) - - @defer.inlineCallbacks - def test_tls_logging(self): - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} - ) - download_handler = build_from_crawler(self.download_handler_cls, crawler) - try: - with LogCapture() as log_capture: - request = Request(self.getURL("file")) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d - log_capture.check_present( - ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) - ) - finally: - yield download_handler.close() - - -class TestSimpleHttps(unittest.TestCase): - """Base class for special cases tested with just one simple request""" - - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - cipher_string: str | None = None - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - self.site = server.Site(r, timeout=None) - self.host = "localhost" - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory( - self.keyfile, self.certfile, cipher_string=self.cipher_string - ), - interface=self.host, - ) - self.portno = self.port.getHost().port - if self.cipher_string is not None: - settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} - else: - settings_dict = None - crawler = get_crawler(settings_dict=settings_dict) - self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"https://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - -class TestHttps11WrongHostname(TestSimpleHttps): - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class TestHttps11InvalidDNSId(TestSimpleHttps): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class TestHttps11InvalidDNSPattern(TestSimpleHttps): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - -class TestHttps11CustomCiphers(TestSimpleHttps): - cipher_string = "CAMELLIA256-SHA" - - -class TestHttp11MockServer(unittest.TestCase): - """HTTP 1.1 test case with MockServer""" - - settings_dict: dict | None = None - is_secure = False - - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid - # download it - yield crawler.crawl( - seed=Request( - url=self.mockserver.url("/partial", is_secure=self.is_secure), - meta={"download_maxsize": 1000}, - ) - ) - failure = crawler.spider.meta["failure"] - assert isinstance(failure.value, defer.CancelledError) - - @defer.inlineCallbacks - def test_download(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl( - seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) - ) - failure = crawler.spider.meta.get("failure") - assert failure is None - reason = crawler.spider.meta["close_reason"] - assert reason == "finished" - - -class UriResource(resource.Resource): - """Return the full uri that was requested""" - - def getChild(self, path, request): - return self - - def render(self, request): - # Note: this is an ugly hack for CONNECT request timeout test. - # Returning some data here fail SSL/TLS handshake - # ToDo: implement proper HTTPS proxy tests, not faking them. - if request.method != b"CONNECT": - return request.uri - return b"" - - -class TestHttpProxy(unittest.TestCase, ABC): - expected_http_proxy_request_body = b"http://example.com" - - @property - @abstractmethod - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - raise NotImplementedError - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - wrapper = WrappingFactory(site) - self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - - def getURL(self, path): - return f"http://127.0.0.1:{self.portno}/{path}" - - def test_download_with_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body - - http_proxy = self.getURL("") - request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_download_without_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == b"/path/to/resource" - - request = Request(self.getURL("path/to/resource")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10Proxy(TestHttpProxy): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - -class TestHttp11Proxy(TestHttpProxy): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): - """Test TunnelingTCP4ClientEndpoint""" - if NON_EXISTING_RESOLVABLE: - raise SkipTest("Non-existing hosts are resolvable") - http_proxy = self.getURL("") - domain = "https://no-such-domain.nosuch" - request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) - d = self.download_request(request, Spider("foo")) - timeout = yield self.assertFailure(d, error.TimeoutError) - assert domain in timeout.osError - - def test_download_with_proxy_without_http_scheme(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body - - http_proxy = self.getURL("").replace("http://", "") - request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): pass diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py new file mode 100644 index 000000000..46e5972f7 --- /dev/null +++ b/tests/test_downloader_handlers_http_base.py @@ -0,0 +1,698 @@ +"""Base classes for HTTP download handler tests.""" + +from __future__ import annotations + +import shutil +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any +from unittest import mock + +import pytest +from testfixtures import LogCapture +from twisted.internet import defer, error, reactor +from twisted.protocols.policies import WrappingFactory +from twisted.trial import unittest +from twisted.web import resource, server, static, util +from twisted.web._newclient import ResponseFailed +from twisted.web.http import _DataLoss + +from scrapy.http import Headers, HtmlResponse, Request, TextResponse +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler +from tests import NON_EXISTING_RESOLVABLE +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) +from tests.spiders import SingleRequestSpider + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class ContentLengthHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the Content-Length + header from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"content-length")[0] + + +class ChunkedResource(resource.Resource): + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +def closeConnection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + request.channel.loseConnection() + request.finish() + + +class EmptyContentTypeHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of request body + without content-type header in response. + """ + + def render(self, request): + request.setHeader("content-type", "") + return request.content.read() + + +class LargeChunkedFileResource(resource.Resource): + def render(self, request): + def response(): + for i in range(1024): + request.write(b"x" * 1024) + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class DuplicateHeaderResource(resource.Resource): + def render(self, request): + request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) + return b"" + + +class TestHttpBase(unittest.TestCase, ABC): + scheme = "http" + + # only used for HTTPS tests + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + r.putChild(b"redirect", util.Redirect(b"/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"chunked", ChunkedResource()) + r.putChild(b"broken-chunked", BrokenChunkedResource()) + r.putChild(b"contentlength", ContentLengthHeaderResource()) + r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + r.putChild(b"largechunkedfile", LargeChunkedFileResource()) + r.putChild(b"duplicate-header", DuplicateHeaderResource()) + r.putChild(b"echo", Echo()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = "localhost" + if self.scheme == "https": + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler + self.port = reactor.listenSSL( + 0, + self.site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host, + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + self.portno = self.port.getHost().port + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() + ) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_download_head(self): + request = Request(self.getURL("file"), method="HEAD") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"") + return d + + def test_redirect_status(self): + request = Request(self.getURL("redirect")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEqual, 302) + return d + + def test_redirect_status_head(self): + request = Request(self.getURL("redirect"), method="HEAD") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEqual, 302) + return d + + @defer.inlineCallbacks + def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest != "default" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + + # client connects but no data is received + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("wait"), meta=meta) + d = self.download_request(request, spider) + yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + @defer.inlineCallbacks + def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest != "default" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects, server send headers and some body bytes but hangs + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("hang-after-headers"), meta=meta) + d = self.download_request(request, spider) + yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + def test_host_header_not_in_request_headers(self): + def _test(response): + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers + + request = Request(self.getURL("host")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_host_header_seted_in_request_headers(self): + host = self.host + ":" + str(self.portno) + + def _test(response): + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() + + request = Request(self.getURL("host"), headers={"Host": host}) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_content_length_zero_bodyless_post_request_headers(self): + """Tests if "Content-Length: 0" is sent for bodyless POST requests. + + This is not strictly required by HTTP RFCs but can cause trouble + for some web servers. + See: + https://github.com/scrapy/scrapy/issues/823 + https://issues.apache.org/jira/browse/TS-2902 + https://github.com/kennethreitz/requests/issues/405 + https://bugs.python.org/issue14721 + """ + + def _test(response): + assert response.body == b"0" + + request = Request(self.getURL("contentlength"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_content_length_zero_bodyless_post_only_one(self): + def _test(response): + import json + + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] + + request = Request(self.getURL("echo"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_payload(self): + body = b"1" * 100 # PayloadResource requires body length to be 100 + request = Request(self.getURL("payload"), method="POST", body=body) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, body) + return d + + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b"content-length"]) + d.addCallback(self.assertEqual, b"159") + return d + + def _test_response_class(self, filename, body, response_class): + def _test(response): + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + + request = Request(self.getURL(filename), body=body) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_response_class_from_url(self): + return self._test_response_class("foo.html", b"", HtmlResponse) + + def test_response_class_from_body(self): + return self._test_response_class( + "foo", + b"\n.", + HtmlResponse, + ) + + def test_get_duplicate_header(self): + def _test(response): + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] + + request = Request(self.getURL("duplicate-header")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + +class TestHttp11Base(TestHttpBase): + """HTTP 1.1 test case""" + + def test_download_without_maxsize_limit(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_response_class_choosing_request(self): + """Tests choosing of correct response type + in case of Content-Type is empty but body contains text. + """ + body = b"Some plain text\ndata with tabs\t and null bytes\0" + + def _test_type(response): + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck + + request = Request(self.getURL("nocontenttype"), body=body) + d = self.download_request(request, Spider("foo")) + d.addCallback(_test_type) + return d + + @defer.inlineCallbacks + def test_download_with_maxsize(self): + request = Request(self.getURL("file")) + + # 10 is minimal size for this request and the limit is only counted on + # response body. (regardless of headers) + d = self.download_request(request, Spider("foo", download_maxsize=10)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + + d = self.download_request(request, Spider("foo", download_maxsize=9)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: + request = Request(self.getURL("largechunkedfile")) + + def check(logger): + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) + + d = self.download_request(request, Spider("foo", download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(0.1, d.callback, logger) + yield d + + @defer.inlineCallbacks + def test_download_with_maxsize_per_req(self): + meta = {"download_maxsize": 2} + request = Request(self.getURL("file"), meta=meta) + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + @defer.inlineCallbacks + def test_download_with_small_maxsize_per_spider(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=2)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + def test_download_with_large_maxsize_per_spider(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=100)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_download_chunked_content(self): + request = Request(self.getURL("chunked")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"chunked content\n") + return d + + def test_download_broken_content_cause_data_loss(self, url="broken"): + # TODO: this one checks for Twisted-specific exceptions + request = Request(self.getURL(url)) + d = self.download_request(request, Spider("foo")) + + def checkDataLoss(failure): + if failure.check(ResponseFailed) and any( + r.check(_DataLoss) for r in failure.value.reasons + ): + return None + return failure + + d.addCallback(lambda _: self.fail("No DataLoss exception")) + d.addErrback(checkDataLoss) + return d + + def test_download_broken_chunked_content_cause_data_loss(self): + return self.test_download_broken_content_cause_data_loss("broken-chunked") + + def test_download_broken_content_allow_data_loss(self, url="broken"): + request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ["dataloss"]) + return d + + def test_download_broken_chunked_content_allow_data_loss(self): + return self.test_download_broken_content_allow_data_loss("broken-chunked") + + def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) + download_handler = build_from_crawler(self.download_handler_cls, crawler) + request = Request(self.getURL(url)) + d = download_handler.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ["dataloss"]) + return d + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + return self.test_download_broken_content_allow_data_loss_via_setting( + "broken-chunked" + ) + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.1") + return d + + +class TestHttps11Base(TestHttp11Base): + scheme = "https" + + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) + + @defer.inlineCallbacks + def test_tls_logging(self): + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} + ) + download_handler = build_from_crawler(self.download_handler_cls, crawler) + try: + with LogCapture() as log_capture: + request = Request(self.getURL("file")) + d = download_handler.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + log_capture.check_present( + ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) + ) + finally: + yield download_handler.close() + + +class TestSimpleHttpsBase(unittest.TestCase, ABC): + """Base class for special cases tested with just one simple request""" + + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + cipher_string: str | None = None + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + self.site = server.Site(r, timeout=None) + self.host = "localhost" + self.port = reactor.listenSSL( + 0, + self.site, + ssl_context_factory( + self.keyfile, self.certfile, cipher_string=self.cipher_string + ), + interface=self.host, + ) + self.portno = self.port.getHost().port + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(settings_dict=settings_dict) + self.download_handler = build_from_crawler(self.download_handler_cls, crawler) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return f"https://{self.host}:{self.portno}/{path}" + + def test_download(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + +class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class TestHttpsInvalidDNSIdBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super().setUp() + self.host = "127.0.0.1" + + +class TestHttpsInvalidDNSPatternBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + +class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): + cipher_string = "CAMELLIA256-SHA" + + +class TestHttpMockServerBase(unittest.TestCase, ABC): + """HTTP 1.1 test case with MockServer""" + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + is_secure = False + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_download_with_content_length(self): + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid + # download it + yield crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) + ) + failure = crawler.spider.meta["failure"] + assert isinstance(failure.value, defer.CancelledError) + + @defer.inlineCallbacks + def test_download(self): + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + yield crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) + failure = crawler.spider.meta.get("failure") + assert failure is None + reason = crawler.spider.meta["close_reason"] + assert reason == "finished" + + +class UriResource(resource.Resource): + """Return the full uri that was requested""" + + def getChild(self, path, request): + return self + + def render(self, request): + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b"CONNECT": + return request.uri + return b"" + + +class TestHttpProxyBase(unittest.TestCase, ABC): + expected_http_proxy_request_body = b"http://example.com" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + wrapper = WrappingFactory(site) + self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") + self.portno = self.port.getHost().port + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() + ) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + + def getURL(self, path): + return f"http://127.0.0.1:{self.portno}/{path}" + + def test_download_with_proxy(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + http_proxy = self.getURL("") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_download_without_proxy(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" + + request = Request(self.getURL("path/to/resource")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + if NON_EXISTING_RESOLVABLE: + pytest.skip("Non-existing hosts are resolvable") + http_proxy = self.getURL("") + domain = "https://no-such-domain.nosuch" + request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) + d = self.download_request(request, Spider("foo")) + timeout = yield self.assertFailure(d, error.TimeoutError) + assert domain in timeout.osError + + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + http_proxy = self.getURL("").replace("http://", "") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test)