RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into throttling

This commit is contained in:
Adrian Chaves 2026-06-26 11:32:43 +02:00
parent 8190b5b703 0007676e8d
commit 3e02b9e5ab
11 changed files with 148 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/topics/settings.rst
View File

@ -751,6 +751,9 @@ necessary to access certain HTTPS websites: for example, you may need to use
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
specific cipher that is not included in ``DEFAULT`` if a website requires it.
Set this setting to ``None`` to use the default ciphers of the underlying TLS
implementation.
.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format
.. note::

11
scrapy/core/downloader/contextfactory.py
View File

@ -17,7 +17,6 @@ from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import (
_TWISTED_VERSION_MAP,
DEFAULT_CIPHERS,
_openssl_methods,
_ScrapyClientTLSOptions,
_ScrapyClientTLSOptions26,
@ -68,11 +67,11 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
self.tls_min_version: TLSVersion | None = tls_min_version
self.tls_max_version: TLSVersion | None = tls_max_version
self.tls_verbose_logging: bool = tls_verbose_logging # unused
self.tls_ciphers: AcceptableCiphers
if tls_ciphers:
self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
else:
self.tls_ciphers = DEFAULT_CIPHERS
self.tls_ciphers: AcceptableCiphers | None = (
AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
if tls_ciphers
else None
)
self._verify_certificates = verify_certificates
@classmethod

5
scrapy/core/downloader/handlers/ftp.py
View File

@ -119,7 +119,10 @@ class FTPDownloadHandler(BaseDownloadHandler):
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
return Response(url=request.url, status=httpcode, body=message.encode())
raise
protocol.close()
finally:
protocol.close()
assert client.transport
client.transport.loseConnection()
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)

12
scrapy/core/downloader/tls.py
View File

@ -43,6 +43,13 @@ _openssl_methods: dict[str, int] = {
def __getattr__(name: str) -> Any:
if name == "DEFAULT_CIPHERS":
warnings.warn(
"scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")
deprecated = {
"METHOD_TLS": "TLS",
"METHOD_TLSv10": "TLSv1.0",
@ -177,8 +184,3 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
return True
return verifyCallback
DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString(
"DEFAULT"
)

52
tests/test_core_downloader.py
View File

@ -8,7 +8,7 @@ import pytest
from pytest_twisted import async_yield_fixture
from twisted.internet.protocol import Factory
from twisted.internet.protocol import Protocol as TxProtocol
from twisted.internet.ssl import optionsForClientTLS
from twisted.internet.ssl import AcceptableCiphers, optionsForClientTLS
from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
@ -180,6 +180,49 @@ class TestContextFactory(TestContextFactoryBase):
assert options & 0x4 # OP_LEGACY_SERVER_CONNECT
class TestContextFactoryCiphers(TestContextFactoryBase):
async def _assert_factory_works(
self, server_url: str, client_context_factory: _ScrapyClientContextFactory
) -> None:
s = "0123456789" * 10
body = await self.get_page(
server_url + "payload", client_context_factory, body=s
)
assert body == to_bytes(s)
def test_default(self) -> None:
"""The default 'DEFAULT' value is passed to Twisted as is."""
crawler = get_crawler()
factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
assert factory.tls_ciphers is not None
# OpenSSLAcceptableCiphers has no __eq__, so compare the parsed ciphers.
assert (
factory.tls_ciphers._ciphers
== AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers
)
assert factory._get_cert_options_kwargs()["acceptableCiphers"] is not None
def test_custom(self) -> None:
crawler = get_crawler(
settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"}
)
factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
assert factory.tls_ciphers is not None
assert (
factory.tls_ciphers._ciphers
== AcceptableCiphers.fromOpenSSLCipherString("CAMELLIA256-SHA")._ciphers
)
@coroutine_test
async def test_none(self, server_url: str) -> None:
"""A None value enables the Twisted default ciphers."""
crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": None})
factory = build_from_crawler(_ScrapyClientContextFactory, crawler)
assert factory.tls_ciphers is None
assert factory._get_cert_options_kwargs()["acceptableCiphers"] is None
await self._assert_factory_works(server_url, factory)
class TestContextFactoryTLSMethod(TestContextFactoryBase):
async def _assert_factory_works(
self, server_url: str, client_context_factory: _ScrapyClientContextFactory
@ -278,3 +321,10 @@ def test_deprecated_tls_module_names() -> None:
match="scrapy.core.downloader.tls.openssl_methods is deprecated",
):
assert isinstance(tls.openssl_methods, dict)
with pytest.warns(
ScrapyDeprecationWarning,
match="scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated",
):
assert tls.DEFAULT_CIPHERS._ciphers == (
AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers
)

53
tests/test_http_cookies.py
View File

@ -1,8 +1,59 @@
from http.cookiejar import DefaultCookiePolicy
from scrapy.http import Request, Response
from scrapy.http.cookies import WrappedRequest, WrappedResponse
from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse
from scrapy.utils.httpobj import urlparse_cached
class TestCookieJar:
def setup_method(self):
self.jar = CookieJar()
self.request = Request("http://example.com/")
self.response = Response(
"http://example.com/",
headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"},
)
def test_extract_cookies(self):
assert len(self.jar) == 0
self.jar.extract_cookies(self.response, self.request)
assert len(self.jar) == 1
cookie = next(iter(self.jar))
assert cookie.name == "name"
assert cookie.value == "value"
assert ".example.com" in self.jar._cookies
def test_make_cookies_and_set_cookie(self):
cookies = self.jar.make_cookies(self.response, self.request)
assert len(cookies) == 1
jar = CookieJar()
for cookie in cookies:
jar.set_cookie(cookie)
assert len(jar) == 1
def test_clear(self):
self.jar.extract_cookies(self.response, self.request)
assert len(self.jar) == 1
self.jar.clear()
assert len(self.jar) == 0
def test_clear_session_cookies(self):
self.jar.extract_cookies(self.response, self.request)
assert len(self.jar) == 1
self.jar.clear_session_cookies()
assert len(self.jar) == 0
def test_set_policy(self):
policy = DefaultCookiePolicy()
self.jar.set_policy(policy)
assert self.jar.jar._policy is policy
def test_check_expired_frequency(self):
jar = CookieJar(check_expired_frequency=1)
jar.add_cookie_header(self.request)
assert jar.processed == 1
class TestWrappedRequest:
def setup_method(self):
self.request = Request(

1
tests/test_http_headers.py
View File

@ -140,6 +140,7 @@ class TestHeaders:
h1["foo"] = "bar"
h1["foo"] = None
h1.setdefault("foo", "bar")
assert h1["foo"] is None
assert h1.get("foo") is None
assert h1.getlist("foo") == []

10
tests/test_http_request.py
View File

@ -24,6 +24,10 @@ class TestRequest:
with pytest.raises(TypeError):
self.request_class(123)
# priority argument must be an integer
with pytest.raises(TypeError, match="Request priority not an integer"):
self.request_class("http://www.example.com", priority="1")
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
assert r.url == "http://www.example.com"
@ -274,6 +278,12 @@ class TestRequest:
assert r4.meta == {}
assert r4.dont_filter is False
# the cls argument allows changing the resulting class
custom_request_cls = type("CustomRequest", (self.request_class,), {})
r5 = r1.replace(cls=custom_request_cls)
assert isinstance(r5, custom_request_cls)
assert r5.url == r1.url
def test_method_always_str(self):
r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)

3
tests/test_http_request_form.py
View File

@ -294,6 +294,9 @@ class TestFormRequest(TestRequest):
assert request.method == "GET"
request = FormRequest.from_response(response, method="POST")
assert request.method == "POST"
# an explicit method=None skips form-method normalization
request = FormRequest.from_response(response, method=None)
assert request.method == "NONE"
def test_from_response_override_url(self):
response = _buildresponse(

5
tests/test_http_response.py
View File

@ -254,6 +254,11 @@ class TestResponse:
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
def test_follow_None_encoding(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="encoding can't be None"):
r.follow("foo", encoding=None)
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",

7
tests/test_http_response_text.py
View File

@ -14,6 +14,13 @@ from tests.test_http_response import TestResponse
class TestTextResponse(TestResponse):
response_class = TextResponse
def test_follow_None_encoding(self):
# unlike the base Response, TextResponse.follow() falls back to the
# response encoding when encoding is None instead of raising
r = self.response_class("http://example.com", body=b"hello", encoding="cp1252")
req = r.follow("foo", encoding=None)
assert req.encoding == "cp1252"
def test_replace(self):
super().test_replace()
r1 = self.response_class(