mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into deprecation-removals
This commit is contained in:
Adrian
GitHub
commit
55c61646da
|
|
@ -12,13 +12,13 @@ Key takeaways:
|
|||
> Note: What follows is based on
|
||||
> https://raw.githubusercontent.com/jackyzha0/quartz/acfaa472253a432d350e9b6904c0cde14f8c487f/.github/pull_request_template.md
|
||||
|
||||
We more than welcome contributions, and are OK with the use of LLMs tools. How
|
||||
We more than welcome contributions, and are OK with the use of LLM tools. How
|
||||
you use those tools depends on whether or not they make you more productive.
|
||||
|
||||
But one thing that bugs us a lot are PRs that are made entirely with these
|
||||
tools, without any revision or any effort trying to refine their output
|
||||
whatsoever. This is just pure laziness, and unacceptable. Doing so will just
|
||||
end up wasting everyone time (ours and yours).
|
||||
end up wasting everyone's time (ours and yours).
|
||||
|
||||
So to be the most productive for all parties, we would encourage any
|
||||
contributors to, at the very least, pay attention to what the model is doing,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from sphinx.application import Sphinx
|
|||
|
||||
def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool:
|
||||
if not skip:
|
||||
# autodocs was generating a text "alias of" for the following members
|
||||
# autodoc was generating the text "alias of" for the following members
|
||||
return name in {"default_item_class", "default_selector_class"}
|
||||
return skip
|
||||
|
||||
|
|
|
|||
|
|
@ -46,12 +46,12 @@ the console you need to type::
|
|||
Password:
|
||||
>>>
|
||||
|
||||
By default Username is ``scrapy`` and Password is autogenerated. The
|
||||
autogenerated Password can be seen on Scrapy logs like the example below::
|
||||
By default, the username is ``scrapy`` and the password is autogenerated. The
|
||||
autogenerated password can be seen on Scrapy logs like the example below::
|
||||
|
||||
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
|
||||
|
||||
Default Username and Password can be overridden by the settings
|
||||
The default username and password can be overridden by the settings
|
||||
:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
|
||||
|
||||
.. warning::
|
||||
|
|
|
|||
|
|
@ -122,18 +122,6 @@ implicit_reexport = true
|
|||
module = "scrapy.settings.default_settings"
|
||||
ignore_errors = true
|
||||
|
||||
# deprecated modules
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"scrapy.core.downloader.webclient",
|
||||
"tests.test_webclient",
|
||||
]
|
||||
allow_any_generics = true
|
||||
allow_untyped_calls = true
|
||||
allow_untyped_defs = true
|
||||
check_untyped_defs = false
|
||||
warn_return_any = false
|
||||
|
||||
# usually no type hints
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
|
|
|
|||
|
|
@ -114,10 +114,10 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
acceptableCiphers=self.tls_ciphers,
|
||||
)
|
||||
|
||||
# kept for old-style HTTP/1.0 downloader context twisted calls,
|
||||
# e.g. connectSSL()
|
||||
# should be removed together with ScrapyClientContextFactory
|
||||
def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context:
|
||||
def getContext(
|
||||
self, hostname: Any = None, port: Any = None
|
||||
) -> SSL.Context: # pragma: no cover
|
||||
return self._get_context()
|
||||
|
||||
def _get_context(self) -> SSL.Context:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# pragma: no file cover
|
||||
import warnings
|
||||
|
||||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http11 import (
|
||||
HTTP11DownloadHandler as HTTPDownloadHandler,
|
||||
)
|
||||
|
|
@ -16,6 +15,5 @@ warnings.warn(
|
|||
)
|
||||
|
||||
__all__ = [
|
||||
"HTTP10DownloadHandler",
|
||||
"HTTPDownloadHandler",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
"""Download handlers for http and https schemes"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from twisted.internet.interfaces import IConnector
|
||||
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Response
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class HTTP10DownloadHandler:
|
||||
lazy = False
|
||||
|
||||
def __init__(self, settings: BaseSettings, crawler: Crawler):
|
||||
warnings.warn(
|
||||
"HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): # pragma: no cover
|
||||
raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
|
||||
self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object(
|
||||
settings["DOWNLOADER_HTTPCLIENTFACTORY"]
|
||||
)
|
||||
if settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL":
|
||||
self.ClientContextFactory: type[_ScrapyClientContextFactory] = (
|
||||
_ScrapyClientContextFactory
|
||||
)
|
||||
else: # pragma: no cover
|
||||
warnings.warn(
|
||||
"The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.ClientContextFactory = load_object(
|
||||
settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
|
||||
)
|
||||
self._settings: BaseSettings = settings
|
||||
self._crawler: Crawler = crawler
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls(crawler.settings, crawler)
|
||||
|
||||
async def download_request(self, request: Request) -> Response:
|
||||
factory = self.HTTPClientFactory(request)
|
||||
self._connect(factory)
|
||||
return await maybe_deferred_to_future(factory.deferred)
|
||||
|
||||
def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector:
|
||||
from twisted.internet import reactor
|
||||
|
||||
host, port = to_unicode(factory.host), factory.port
|
||||
if factory.scheme == b"https":
|
||||
client_context_factory = build_from_crawler(
|
||||
self.ClientContextFactory,
|
||||
self._crawler,
|
||||
)
|
||||
return reactor.connectSSL(host, port, factory, client_context_factory)
|
||||
return reactor.connectTCP(host, port, factory)
|
||||
|
||||
async def close(self) -> None:
|
||||
pass
|
||||
|
|
@ -1,243 +0,0 @@
|
|||
"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from time import monotonic, time
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urldefrag, urlparse, urlunparse
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.protocol import ClientFactory
|
||||
from twisted.web.http import HTTPClient
|
||||
|
||||
from scrapy.exceptions import DownloadTimeoutError, ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Request
|
||||
|
||||
|
||||
class ScrapyHTTPPageGetter(HTTPClient):
|
||||
delimiter = b"\n"
|
||||
|
||||
def __init__(self):
|
||||
warnings.warn(
|
||||
"ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
super().__init__()
|
||||
|
||||
def connectionMade(self):
|
||||
self.headers = Headers() # bucket for response headers
|
||||
|
||||
# Method command
|
||||
self.sendCommand(self.factory.method, self.factory.path)
|
||||
# Headers
|
||||
for key, values in self.factory.headers.items():
|
||||
for value in values:
|
||||
self.sendHeader(key, value)
|
||||
self.endHeaders()
|
||||
# Body
|
||||
if self.factory.body is not None:
|
||||
self.transport.write(self.factory.body)
|
||||
|
||||
def lineReceived(self, line):
|
||||
return HTTPClient.lineReceived(self, line.rstrip())
|
||||
|
||||
def handleHeader(self, key, value):
|
||||
self.headers.appendlist(key, value)
|
||||
|
||||
def handleStatus(self, version, status, message):
|
||||
self.factory.gotStatus(version, status, message)
|
||||
|
||||
def handleEndHeaders(self):
|
||||
self.factory.gotHeaders(self.headers)
|
||||
|
||||
def connectionLost(self, reason):
|
||||
self._connection_lost_reason = reason
|
||||
HTTPClient.connectionLost(self, reason)
|
||||
self.factory.noPage(reason)
|
||||
|
||||
def handleResponse(self, response):
|
||||
if self.factory.method.upper() == b"HEAD":
|
||||
self.factory.page(b"")
|
||||
elif self.length is not None and self.length > 0:
|
||||
self.factory.noPage(self._connection_lost_reason)
|
||||
else:
|
||||
self.factory.page(response)
|
||||
self.transport.loseConnection()
|
||||
|
||||
def timeout(self):
|
||||
self.transport.loseConnection()
|
||||
|
||||
# transport cleanup needed for HTTPS connections
|
||||
if self.factory.url.startswith(b"https"):
|
||||
self.transport.stopProducing()
|
||||
|
||||
self.factory.noPage(
|
||||
DownloadTimeoutError(
|
||||
f"Getting {self.factory.url} took longer "
|
||||
f"than {self.factory.timeout} seconds."
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# This class used to inherit from Twisted’s
|
||||
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
|
||||
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
|
||||
# non-overridden code into this class.
|
||||
class ScrapyHTTPClientFactory(ClientFactory):
|
||||
protocol = ScrapyHTTPPageGetter
|
||||
|
||||
waiting = 1
|
||||
noisy = False
|
||||
followRedirect = False
|
||||
afterFoundGet = False
|
||||
|
||||
def _build_response(self, body, request):
|
||||
request.meta["download_latency"] = (
|
||||
self._headers_time_mono - self._start_time_mono
|
||||
)
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
|
||||
return respcls(
|
||||
url=self._url,
|
||||
status=status,
|
||||
headers=headers,
|
||||
body=body,
|
||||
protocol=to_unicode(self.version),
|
||||
)
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
proxy = request.meta.get("proxy")
|
||||
if proxy:
|
||||
proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii"))
|
||||
self.scheme = proxy_parsed.scheme
|
||||
self.host = proxy_parsed.hostname
|
||||
self.port = proxy_parsed.port
|
||||
self.netloc = proxy_parsed.netloc
|
||||
if self.port is None:
|
||||
self.port = 443 if proxy_parsed.scheme == b"https" else 80
|
||||
self.path = self.url
|
||||
else:
|
||||
parsed = urlparse_cached(request)
|
||||
path_str = urlunparse(
|
||||
("", "", parsed.path or "/", parsed.params, parsed.query, "")
|
||||
)
|
||||
self.path = to_bytes(path_str, encoding="ascii")
|
||||
assert parsed.hostname is not None
|
||||
self.host = to_bytes(parsed.hostname, encoding="ascii")
|
||||
self.port = parsed.port
|
||||
self.scheme = to_bytes(parsed.scheme, encoding="ascii")
|
||||
self.netloc = to_bytes(parsed.netloc, encoding="ascii")
|
||||
if self.port is None:
|
||||
self.port = 443 if self.scheme == b"https" else 80
|
||||
|
||||
def __init__(self, request: Request, timeout: float = 180):
|
||||
warnings.warn(
|
||||
"ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
self._url: str = urldefrag(request.url)[0]
|
||||
# converting to bytes to comply to Twisted interface
|
||||
self.url: bytes = to_bytes(self._url, encoding="ascii")
|
||||
self.method: bytes = to_bytes(request.method, encoding="ascii")
|
||||
self.body: bytes | None = request.body or None
|
||||
self.headers: Headers = Headers(request.headers)
|
||||
self.response_headers: Headers | None = None
|
||||
self.timeout: float = request.meta.get("download_timeout") or timeout
|
||||
self.start_time: float = time()
|
||||
self._start_time_mono: float = monotonic()
|
||||
self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
|
||||
self._build_response, request
|
||||
)
|
||||
|
||||
# Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected
|
||||
# to have _disconnectedDeferred. See Twisted r32329.
|
||||
# As Scrapy implements it's own logic to handle redirects is not
|
||||
# needed to add the callback _waitForDisconnect.
|
||||
# Specifically this avoids the AttributeError exception when
|
||||
# clientConnectionFailed method is called.
|
||||
self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred()
|
||||
|
||||
self._set_connection_attributes(request)
|
||||
|
||||
# set Host header based on url
|
||||
self.headers.setdefault("Host", self.netloc)
|
||||
|
||||
# set Content-Length based len of body
|
||||
if self.body is not None:
|
||||
self.headers["Content-Length"] = len(self.body)
|
||||
# just in case a broken http/1.1 decides to keep connection alive
|
||||
self.headers.setdefault("Connection", "close")
|
||||
# Content-Length must be specified in POST method even with no body
|
||||
elif self.method == b"POST":
|
||||
self.headers["Content-Length"] = 0
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<{self.__class__.__name__}: {self._url}>"
|
||||
|
||||
def _cancelTimeout(self, result, timeoutCall):
|
||||
if timeoutCall.active():
|
||||
timeoutCall.cancel()
|
||||
return result
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = ClientFactory.buildProtocol(self, addr)
|
||||
p.followRedirect = self.followRedirect
|
||||
p.afterFoundGet = self.afterFoundGet
|
||||
if self.timeout:
|
||||
from twisted.internet import reactor
|
||||
|
||||
timeoutCall = reactor.callLater(self.timeout, p.timeout)
|
||||
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
|
||||
return p
|
||||
|
||||
def gotHeaders(self, headers):
|
||||
self.headers_time = time()
|
||||
self._headers_time_mono = monotonic()
|
||||
self.response_headers = headers
|
||||
|
||||
def gotStatus(self, version, status, message):
|
||||
"""
|
||||
Set the status of the request on us.
|
||||
@param version: The HTTP version.
|
||||
@type version: L{bytes}
|
||||
@param status: The HTTP status code, an integer represented as a
|
||||
bytestring.
|
||||
@type status: L{bytes}
|
||||
@param message: The HTTP status message.
|
||||
@type message: L{bytes}
|
||||
"""
|
||||
self.version, self.status, self.message = version, status, message
|
||||
|
||||
def page(self, page):
|
||||
if self.waiting:
|
||||
self.waiting = 0
|
||||
self.deferred.callback(page)
|
||||
|
||||
def noPage(self, reason):
|
||||
if self.waiting:
|
||||
self.waiting = 0
|
||||
self.deferred.errback(reason)
|
||||
|
||||
def clientConnectionFailed(self, _, reason):
|
||||
"""
|
||||
When a connection attempt fails, the request cannot be issued. If no
|
||||
result has yet been provided to the result Deferred, provide the
|
||||
connection failure reason as an error result.
|
||||
"""
|
||||
if self.waiting:
|
||||
self.waiting = 0
|
||||
# If the connection attempt failed, there is nothing more to
|
||||
# disconnect, so just fire that Deferred now.
|
||||
self._disconnectedDeferred.callback(None)
|
||||
self.deferred.errback(reason)
|
||||
|
|
@ -54,7 +54,6 @@ __all__ = [
|
|||
"DOWNLOADER_CLIENT_TLS_CIPHERS",
|
||||
"DOWNLOADER_CLIENT_TLS_METHOD",
|
||||
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING",
|
||||
"DOWNLOADER_HTTPCLIENTFACTORY",
|
||||
"DOWNLOADER_MIDDLEWARES",
|
||||
"DOWNLOADER_MIDDLEWARES_BASE",
|
||||
"DOWNLOADER_STATS",
|
||||
|
|
@ -275,10 +274,6 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
|
|||
DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
|
||||
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
|
||||
|
||||
DOWNLOADER_HTTPCLIENTFACTORY = (
|
||||
"scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
|
||||
)
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {}
|
||||
DOWNLOADER_MIDDLEWARES_BASE = {
|
||||
# Engine side
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ import pytest
|
|||
|
||||
from scrapy import Request
|
||||
from tests.test_downloader_handlers_http_base import (
|
||||
TestHttp11Base,
|
||||
TestHttpBase,
|
||||
TestHttpProxyBase,
|
||||
TestHttps11Base,
|
||||
TestHttpsBase,
|
||||
TestHttpsCustomCiphersBase,
|
||||
TestHttpsInvalidDNSIdBase,
|
||||
TestHttpsInvalidDNSPatternBase,
|
||||
|
|
@ -42,7 +42,7 @@ class HttpxDownloadHandlerMixin:
|
|||
return HttpxDownloadHandler
|
||||
|
||||
|
||||
class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
|
||||
class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
|
||||
handler_supports_bindaddress_meta = False
|
||||
|
||||
@pytest.mark.skipif(
|
||||
|
|
@ -77,7 +77,7 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
|
|||
)
|
||||
|
||||
|
||||
class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base):
|
||||
class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
|
||||
handler_supports_bindaddress_meta = False
|
||||
tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
|
||||
|
||||
|
|
@ -90,25 +90,25 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
|
|||
pass
|
||||
|
||||
|
||||
class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11InvalidDNSPattern(
|
||||
class TestHttpsInvalidDNSPattern(
|
||||
HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
|
||||
class TestHttpWithCrawler(TestHttpWithCrawlerBase):
|
||||
@property
|
||||
def settings_dict(self) -> dict[str, Any] | None:
|
||||
return {
|
||||
|
|
@ -119,7 +119,7 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
|
|||
}
|
||||
|
||||
|
||||
class TestHttps11WithCrawler(TestHttp11WithCrawler):
|
||||
class TestHttpsWithCrawler(TestHttpWithCrawler):
|
||||
is_secure = True
|
||||
|
||||
@pytest.mark.skip(reason="response.certificate is not implemented")
|
||||
|
|
@ -129,10 +129,10 @@ class TestHttps11WithCrawler(TestHttp11WithCrawler):
|
|||
|
||||
|
||||
@pytest.mark.skip(reason="Proxy support is not implemented yet")
|
||||
class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
|
||||
class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Proxy support is not implemented yet")
|
||||
class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
|
||||
class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
|
||||
is_secure = True
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f
|
||||
from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
|
||||
pytestmark = pytest.mark.requires_reactor # HTTP10DownloadHandler requires a reactor
|
||||
|
||||
|
||||
class HTTP10DownloadHandlerMixin:
|
||||
@property
|
||||
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
|
||||
return HTTP10DownloadHandler
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase):
|
||||
"""HTTP 1.0 test case"""
|
||||
|
||||
def test_unsupported_scheme(self) -> None: # type: ignore[override]
|
||||
pytest.skip("Check not implemented")
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_protocol(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/host", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.protocol == "HTTP/1.0"
|
||||
|
||||
|
||||
class TestHttps10(TestHttp10):
|
||||
is_secure = True
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase):
|
||||
@deferred_f_from_coro_f
|
||||
async def test_download_with_proxy_https_timeout(self):
|
||||
pytest.skip("Not implemented")
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_download_with_proxy_without_http_scheme(self):
|
||||
pytest.skip("Not implemented")
|
||||
|
|
@ -11,9 +11,9 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
|||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from tests.test_downloader_handlers_http_base import (
|
||||
TestHttp11Base,
|
||||
TestHttpBase,
|
||||
TestHttpProxyBase,
|
||||
TestHttps11Base,
|
||||
TestHttpsBase,
|
||||
TestHttpsCustomCiphersBase,
|
||||
TestHttpsInvalidDNSIdBase,
|
||||
TestHttpsInvalidDNSPatternBase,
|
||||
|
|
@ -41,11 +41,11 @@ def test_not_configured_without_reactor() -> None:
|
|||
HTTP11DownloadHandler.from_crawler(crawler)
|
||||
|
||||
|
||||
class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base):
|
||||
class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base):
|
||||
class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase):
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -53,25 +53,25 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase):
|
|||
pass
|
||||
|
||||
|
||||
class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11InvalidDNSPattern(
|
||||
class TestHttpsInvalidDNSPattern(
|
||||
HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
|
||||
class TestHttpWithCrawler(TestHttpWithCrawlerBase):
|
||||
@property
|
||||
def settings_dict(self) -> dict[str, Any] | None:
|
||||
return {
|
||||
|
|
@ -82,9 +82,9 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
|
|||
}
|
||||
|
||||
|
||||
class TestHttps11WithCrawler(TestHttp11WithCrawler):
|
||||
class TestHttpsWithCrawler(TestHttpWithCrawler):
|
||||
is_secure = True
|
||||
|
||||
|
||||
class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
|
||||
class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from scrapy.http import Request
|
|||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from tests.test_downloader_handlers_http_base import (
|
||||
TestHttpProxyBase,
|
||||
TestHttps11Base,
|
||||
TestHttpsBase,
|
||||
TestHttpsCustomCiphersBase,
|
||||
TestHttpsInvalidDNSIdBase,
|
||||
TestHttpsInvalidDNSPatternBase,
|
||||
|
|
@ -61,7 +61,7 @@ def test_not_configured_without_reactor() -> None:
|
|||
H2DownloadHandler.from_crawler(crawler)
|
||||
|
||||
|
||||
class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
||||
class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
|
||||
http2 = True
|
||||
handler_supports_http2_dataloss = False
|
||||
|
||||
|
|
@ -149,27 +149,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
await download_handler.download_request(request)
|
||||
|
||||
|
||||
class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
class TestHttp2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps2InvalidDNSPattern(
|
||||
class TestHttp2InvalidDNSPattern(
|
||||
H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase):
|
||||
pass
|
||||
|
||||
|
||||
class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
|
||||
"""HTTP 2.0 test case with MockServer"""
|
||||
|
||||
@property
|
||||
def settings_dict(self) -> dict[str, Any] | None:
|
||||
return {
|
||||
|
|
@ -194,7 +192,7 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
|
|||
pytest.skip("headers_received support is not implemented")
|
||||
|
||||
|
||||
class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
|
||||
class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
|
||||
is_secure = True
|
||||
expected_http_proxy_request_body = b"/"
|
||||
|
||||
|
|
|
|||
|
|
@ -55,9 +55,17 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class TestHttpBase(ABC):
|
||||
is_secure = False
|
||||
is_secure: bool = False
|
||||
http2: bool = False
|
||||
# whether the handler supports per-request bindaddress
|
||||
handler_supports_bindaddress_meta = True
|
||||
handler_supports_bindaddress_meta: bool = True
|
||||
# RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a
|
||||
# stream error (of type PROTOCOL_ERROR) so the client will send
|
||||
# RST_STREAM. Some libraries do only this while e.g. h2 also closes the
|
||||
# connection (see handling of ProtocolError in
|
||||
# h2.connection.H2Connection.receive_data()), thus closing all streams that
|
||||
# were using it, and we handle this as a normal exception.
|
||||
handler_supports_http2_dataloss: bool = True
|
||||
# default headers added by the underlying library that cannot be suppressed
|
||||
always_present_req_headers: ClassVar[frozenset[str]] = frozenset()
|
||||
|
||||
|
|
@ -519,17 +527,6 @@ class TestHttpBase(ABC):
|
|||
else:
|
||||
assert latency > 0
|
||||
|
||||
|
||||
class TestHttp11Base(TestHttpBase):
|
||||
http2: bool = False
|
||||
# RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a
|
||||
# stream error (of type PROTOCOL_ERROR) so the client will send
|
||||
# RST_STREAM. Some libraries do only this while e.g. h2 also closes the
|
||||
# connection (see handling of ProtocolError in
|
||||
# h2.connection.H2Connection.receive_data()), thus closing all streams that
|
||||
# were using it, and we handle this as a normal exception.
|
||||
handler_supports_http2_dataloss: bool = True
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/text", is_secure=self.is_secure))
|
||||
|
|
@ -803,7 +800,7 @@ class TestHttp11Base(TestHttpBase):
|
|||
)
|
||||
|
||||
|
||||
class TestHttps11Base(TestHttp11Base):
|
||||
class TestHttpsBase(TestHttpBase):
|
||||
is_secure = True
|
||||
|
||||
tls_log_message = (
|
||||
|
|
|
|||
|
|
@ -1,404 +0,0 @@
|
|||
"""
|
||||
Tests borrowed from the twisted.web.client tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import OpenSSL.SSL
|
||||
import pytest
|
||||
from pytest_twisted import async_yield_fixture
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.internet.testing import StringTransport
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.web import resource, server, static, util
|
||||
from twisted.web.client import _makeGetterFactory
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory
|
||||
from scrapy.exceptions import DownloadTimeoutError
|
||||
from scrapy.http import Headers, Request
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver.http_resources import (
|
||||
ForeverTakingResource,
|
||||
HostHeaderResource,
|
||||
PayloadResource,
|
||||
)
|
||||
from tests.mockserver.utils import ssl_context_factory
|
||||
from tests.test_core_downloader import TestContextFactoryBase
|
||||
|
||||
# these tests are related to the Twisted HTTP code
|
||||
pytestmark = pytest.mark.requires_reactor
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
|
||||
"""Adapted version of twisted.web.client.getPage"""
|
||||
|
||||
def _clientfactory(url, *args, **kwargs):
|
||||
url = to_unicode(url)
|
||||
timeout = kwargs.pop("timeout", 0)
|
||||
f = client.ScrapyHTTPClientFactory(
|
||||
Request(url, *args, **kwargs), timeout=timeout
|
||||
)
|
||||
f.deferred.addCallback(response_transform or (lambda r: r.body))
|
||||
return f
|
||||
|
||||
return _makeGetterFactory(
|
||||
to_bytes(url),
|
||||
_clientfactory,
|
||||
*args,
|
||||
contextFactory=contextFactory,
|
||||
**kwargs,
|
||||
).deferred
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestScrapyHTTPPageGetter:
|
||||
def test_earlyHeaders(self):
|
||||
# basic test stolen from twisted HTTPageGetter
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
Request(
|
||||
url="http://foo/bar",
|
||||
body="some data",
|
||||
headers={
|
||||
"Host": "example.net",
|
||||
"User-Agent": "fooble",
|
||||
"Cookie": "blah blah",
|
||||
"Content-Length": "12981",
|
||||
"Useful": "value",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
self._test(
|
||||
factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
b"Content-Length: 9\r\n"
|
||||
b"Useful: value\r\n"
|
||||
b"Connection: close\r\n"
|
||||
b"User-Agent: fooble\r\n"
|
||||
b"Host: example.net\r\n"
|
||||
b"Cookie: blah blah\r\n"
|
||||
b"\r\n"
|
||||
b"some data",
|
||||
)
|
||||
|
||||
# test minimal sent headers
|
||||
factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar"))
|
||||
self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n")
|
||||
|
||||
# test a simple POST with body and content-type
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
Request(
|
||||
method="POST",
|
||||
url="http://foo/bar",
|
||||
body="name=value",
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
)
|
||||
|
||||
self._test(
|
||||
factory,
|
||||
b"POST /bar HTTP/1.0\r\n"
|
||||
b"Host: foo\r\n"
|
||||
b"Connection: close\r\n"
|
||||
b"Content-Type: application/x-www-form-urlencoded\r\n"
|
||||
b"Content-Length: 10\r\n"
|
||||
b"\r\n"
|
||||
b"name=value",
|
||||
)
|
||||
|
||||
# test a POST method with no body provided
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
Request(method="POST", url="http://foo/bar")
|
||||
)
|
||||
|
||||
self._test(
|
||||
factory,
|
||||
b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n",
|
||||
)
|
||||
|
||||
# test with single and multivalued headers
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
Request(
|
||||
url="http://foo/bar",
|
||||
headers={
|
||||
"X-Meta-Single": "single",
|
||||
"X-Meta-Multivalued": ["value1", "value2"],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
self._test(
|
||||
factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
b"Host: foo\r\n"
|
||||
b"X-Meta-Multivalued: value1\r\n"
|
||||
b"X-Meta-Multivalued: value2\r\n"
|
||||
b"X-Meta-Single: single\r\n"
|
||||
b"\r\n",
|
||||
)
|
||||
|
||||
# same test with single and multivalued headers but using Headers class
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
Request(
|
||||
url="http://foo/bar",
|
||||
headers=Headers(
|
||||
{
|
||||
"X-Meta-Single": "single",
|
||||
"X-Meta-Multivalued": ["value1", "value2"],
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
self._test(
|
||||
factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
b"Host: foo\r\n"
|
||||
b"X-Meta-Multivalued: value1\r\n"
|
||||
b"X-Meta-Multivalued: value2\r\n"
|
||||
b"X-Meta-Single: single\r\n"
|
||||
b"\r\n",
|
||||
)
|
||||
|
||||
def _test(self, factory, testvalue):
|
||||
transport = StringTransport()
|
||||
protocol = client.ScrapyHTTPPageGetter()
|
||||
protocol.factory = factory
|
||||
protocol.makeConnection(transport)
|
||||
assert set(transport.value().splitlines()) == set(testvalue.splitlines())
|
||||
return testvalue
|
||||
|
||||
def test_non_standard_line_endings(self):
|
||||
# regression test for: http://dev.scrapy.org/ticket/258
|
||||
factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar"))
|
||||
protocol = client.ScrapyHTTPPageGetter()
|
||||
protocol.factory = factory
|
||||
protocol.headers = Headers()
|
||||
protocol.dataReceived(b"HTTP/1.0 200 OK\n")
|
||||
protocol.dataReceived(b"Hello: World\n")
|
||||
protocol.dataReceived(b"Foo: Bar\n")
|
||||
protocol.dataReceived(b"\n")
|
||||
assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]})
|
||||
|
||||
|
||||
class EncodingResource(resource.Resource):
|
||||
out_encoding = "cp1251"
|
||||
|
||||
def render(self, request):
|
||||
body = to_unicode(request.content.read())
|
||||
request.setHeader(b"content-encoding", self.out_encoding)
|
||||
return body.encode(self.out_encoding)
|
||||
|
||||
|
||||
class BrokenDownloadResource(resource.Resource):
|
||||
def render(self, request):
|
||||
# only sends 3 bytes even though it claims to send 5
|
||||
request.setHeader(b"content-length", b"5")
|
||||
request.write(b"abc")
|
||||
return b""
|
||||
|
||||
|
||||
class ErrorResource(resource.Resource):
|
||||
def render(self, request):
|
||||
request.setResponseCode(401)
|
||||
if request.args.get(b"showlength"):
|
||||
request.setHeader(b"content-length", b"0")
|
||||
return b""
|
||||
|
||||
|
||||
class NoLengthResource(resource.Resource):
|
||||
def render(self, request):
|
||||
return b"nolength"
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestWebClient:
|
||||
def _listen(self, site):
|
||||
from twisted.internet import reactor
|
||||
|
||||
return reactor.listenTCP(0, site, interface="127.0.0.1")
|
||||
|
||||
@pytest.fixture
|
||||
def wrapper(self, tmp_path):
|
||||
(tmp_path / "file").write_bytes(b"0123456789")
|
||||
r = static.File(str(tmp_path))
|
||||
r.putChild(b"redirect", util.Redirect(b"/file"))
|
||||
r.putChild(b"wait", ForeverTakingResource())
|
||||
r.putChild(b"error", ErrorResource())
|
||||
r.putChild(b"nolength", NoLengthResource())
|
||||
r.putChild(b"host", HostHeaderResource())
|
||||
r.putChild(b"payload", PayloadResource())
|
||||
r.putChild(b"broken", BrokenDownloadResource())
|
||||
r.putChild(b"encoding", EncodingResource())
|
||||
site = server.Site(r, timeout=None)
|
||||
return WrappingFactory(site)
|
||||
|
||||
@async_yield_fixture
|
||||
async def server_port(self, wrapper):
|
||||
port = self._listen(wrapper)
|
||||
|
||||
yield port.getHost().port
|
||||
|
||||
await port.stopListening()
|
||||
|
||||
@pytest.fixture
|
||||
def server_url(self, server_port):
|
||||
return f"http://127.0.0.1:{server_port}/"
|
||||
|
||||
@inlineCallbacks
|
||||
def testPayload(self, server_url):
|
||||
s = "0123456789" * 10
|
||||
body = yield getPage(server_url + "payload", body=s)
|
||||
assert body == to_bytes(s)
|
||||
|
||||
@inlineCallbacks
|
||||
def testHostHeader(self, server_port, server_url):
|
||||
# if we pass Host header explicitly, it should be used, otherwise
|
||||
# it should extract from url
|
||||
body = yield getPage(server_url + "host")
|
||||
assert body == to_bytes(f"127.0.0.1:{server_port}")
|
||||
body = yield getPage(server_url + "host", headers={"Host": "www.example.com"})
|
||||
assert body == to_bytes("www.example.com")
|
||||
|
||||
@inlineCallbacks
|
||||
def test_getPage(self, server_url):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the body of the response if the default method B{GET} is used.
|
||||
"""
|
||||
body = yield getPage(server_url + "file")
|
||||
assert body == b"0123456789"
|
||||
|
||||
@inlineCallbacks
|
||||
def test_getPageHead(self, server_url):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the empty string if the method is C{HEAD} and there is a successful
|
||||
response code.
|
||||
"""
|
||||
|
||||
def _getPage(method):
|
||||
return getPage(server_url + "file", method=method)
|
||||
|
||||
body = yield _getPage("head")
|
||||
assert body == b""
|
||||
body = yield _getPage("HEAD")
|
||||
assert body == b""
|
||||
|
||||
@inlineCallbacks
|
||||
def test_timeoutNotTriggering(self, server_port, server_url):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and the page is
|
||||
retrieved before the timeout period elapses, the L{Deferred} is
|
||||
called back with the contents of the page.
|
||||
"""
|
||||
body = yield getPage(server_url + "host", timeout=100)
|
||||
assert body == to_bytes(f"127.0.0.1:{server_port}")
|
||||
|
||||
@inlineCallbacks
|
||||
def test_timeoutTriggering(self, wrapper, server_url):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and that many
|
||||
seconds elapse before the server responds to the request. the
|
||||
L{Deferred} is errbacked with a L{DownloadTimeoutError}.
|
||||
"""
|
||||
with pytest.raises(DownloadTimeoutError):
|
||||
yield getPage(server_url + "wait", timeout=0.000001)
|
||||
# Clean up the server which is hanging around not doing
|
||||
# anything.
|
||||
connected = list(wrapper.protocols.keys())
|
||||
# There might be nothing here if the server managed to already see
|
||||
# that the connection was lost.
|
||||
if connected:
|
||||
connected[0].transport.loseConnection()
|
||||
|
||||
@inlineCallbacks
|
||||
def testNotFound(self, server_url):
|
||||
body = yield getPage(server_url + "notsuchfile")
|
||||
assert b"404 - No Such Resource" in body
|
||||
|
||||
@inlineCallbacks
|
||||
def testFactoryInfo(self, server_url):
|
||||
from twisted.internet import reactor
|
||||
|
||||
url = server_url + "file"
|
||||
parsed = urlparse(url)
|
||||
factory = client.ScrapyHTTPClientFactory(Request(url))
|
||||
reactor.connectTCP(parsed.hostname, parsed.port, factory)
|
||||
yield factory.deferred
|
||||
assert factory.status == b"200"
|
||||
assert factory.version.startswith(b"HTTP/")
|
||||
assert factory.message == b"OK"
|
||||
assert factory.response_headers[b"content-length"] == b"10"
|
||||
|
||||
@inlineCallbacks
|
||||
def testRedirect(self, server_url):
|
||||
body = yield getPage(server_url + "redirect")
|
||||
assert (
|
||||
body
|
||||
== b'\n<html>\n <head>\n <meta http-equiv="refresh" content="0;URL=/file">\n'
|
||||
b' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
|
||||
b'<a href="/file">click here</a>\n </body>\n</html>\n'
|
||||
)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_encoding(self, server_url):
|
||||
"""Test that non-standart body encoding matches
|
||||
Content-Encoding header"""
|
||||
original_body = b"\xd0\x81\xd1\x8e\xd0\xaf"
|
||||
response = yield getPage(
|
||||
server_url + "encoding", body=original_body, response_transform=lambda r: r
|
||||
)
|
||||
content_encoding = to_unicode(response.headers[b"Content-Encoding"])
|
||||
assert content_encoding == EncodingResource.out_encoding
|
||||
assert response.body.decode(content_encoding) == to_unicode(original_body)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestWebClientSSL(TestContextFactoryBase):
|
||||
@inlineCallbacks
|
||||
def testPayload(self, server_url):
|
||||
s = "0123456789" * 10
|
||||
body = yield getPage(server_url + "payload", body=s)
|
||||
assert body == to_bytes(s)
|
||||
|
||||
|
||||
class TestWebClientCustomCiphersSSL(TestWebClientSSL):
|
||||
# we try to use a cipher that is not enabled by default in OpenSSL
|
||||
custom_ciphers = "CAMELLIA256-SHA"
|
||||
context_factory = ssl_context_factory(cipher_string=custom_ciphers)
|
||||
|
||||
@inlineCallbacks
|
||||
def testPayload(self, server_url):
|
||||
s = "0123456789" * 10
|
||||
crawler = get_crawler(
|
||||
settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}
|
||||
)
|
||||
client_context_factory = build_from_crawler(
|
||||
_ScrapyClientContextFactory, crawler
|
||||
)
|
||||
body = yield getPage(
|
||||
server_url + "payload", body=s, contextFactory=client_context_factory
|
||||
)
|
||||
assert body == to_bytes(s)
|
||||
|
||||
@inlineCallbacks
|
||||
def testPayloadDisabledCipher(self, server_url):
|
||||
s = "0123456789" * 10
|
||||
crawler = get_crawler(
|
||||
settings_dict={
|
||||
"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"
|
||||
}
|
||||
)
|
||||
client_context_factory = build_from_crawler(
|
||||
_ScrapyClientContextFactory, crawler
|
||||
)
|
||||
with pytest.raises(OpenSSL.SSL.Error):
|
||||
yield getPage(
|
||||
server_url + "payload", body=s, contextFactory=client_context_factory
|
||||
)
|
||||
Loading…
Reference in New Issue