mirror of https://github.com/scrapy/scrapy.git
Add support for HTTP/2 and for SOCKS proxies to HttpxDownloadHandler, improve handler docs (#7575)
* Add support for HTTP/2 and SOCKS proxies to HttpxDownloadHandler. * Update the docs. * Trim the tables. * Restore lost wording. * Handlers docs improvements and fixes.
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
d2290c35c2
commit
4e956bd2de
13
conftest.py
13
conftest.py
|
|
@ -99,6 +99,19 @@ def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmPr
|
|||
proxy.stop()
|
||||
|
||||
|
||||
@pytest.fixture # function scope because it modifies os.environ
|
||||
def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
|
||||
proxy = MitmProxy(mode="socks5")
|
||||
url = proxy.start()
|
||||
monkeypatch.setenv("http_proxy", url)
|
||||
monkeypatch.setenv("https_proxy", url)
|
||||
|
||||
try:
|
||||
yield proxy
|
||||
finally:
|
||||
proxy.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def reactor_pytest(request) -> str:
|
||||
return request.config.getoption("--reactor")
|
||||
|
|
|
|||
17
docs/faq.rst
17
docs/faq.rst
|
|
@ -82,10 +82,18 @@ to steal from us!
|
|||
Does Scrapy work with HTTP proxies?
|
||||
-----------------------------------
|
||||
|
||||
Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP
|
||||
Proxy downloader middleware. See
|
||||
Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader
|
||||
middleware. See
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.
|
||||
|
||||
Does Scrapy work with SOCKS proxies?
|
||||
------------------------------------
|
||||
|
||||
Yes, when using
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the
|
||||
handler documentation.
|
||||
|
||||
How can I scrape an item with attributes in different pages?
|
||||
------------------------------------------------------------
|
||||
|
||||
|
|
@ -360,7 +368,10 @@ method for this purpose. For example:
|
|||
Does Scrapy support IPv6 addresses?
|
||||
-----------------------------------
|
||||
|
||||
Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
Yes, but when using
|
||||
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to
|
||||
set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
Note that by doing so, you lose the ability to set a specific timeout for DNS requests
|
||||
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
|
||||
|
||||
|
|
|
|||
|
|
@ -130,44 +130,39 @@ these exceptions.
|
|||
|
||||
.. _download-handlers-ref:
|
||||
|
||||
Built-in download handlers reference
|
||||
====================================
|
||||
Built-in HTTP download handlers reference
|
||||
=========================================
|
||||
|
||||
DataURIDownloadHandler
|
||||
----------------------
|
||||
Scrapy ships several handlers for HTTP and HTTPS requests. While all of them
|
||||
support basic features, they may differ in support of specific Scrapy features
|
||||
and settings and HTTP protocol features. See the documentation of specific
|
||||
handlers and specific settings for more information. Additionally, as the
|
||||
underlying HTTP client implementations differ between handlers, the behavior of
|
||||
specific websites may be different when doing the same Scrapy requests but
|
||||
using different handlers.
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
|
||||
Here is a comparison of some features of the built-in HTTP handlers, see the
|
||||
individual handler docs for more differences:
|
||||
|
||||
| Supported scheme: ``data``.
|
||||
| Lazy: no.
|
||||
================== ================= ===================== ====================
|
||||
Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
|
||||
================== ================= ===================== ====================
|
||||
Requires asyncio No No Yes
|
||||
Requires a reactor Yes Yes No
|
||||
HTTP/1.1 No Yes Yes
|
||||
HTTP/2 Yes No Yes
|
||||
TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``
|
||||
HTTP proxies No Yes Yes
|
||||
SOCKS proxies No No Yes
|
||||
================== ================= ===================== ====================
|
||||
|
||||
This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
|
||||
You can find additional HTTP download handlers in the
|
||||
scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
|
||||
developers and contains experimental handlers that may be included in some
|
||||
later Scrapy version but can already be used. Please refer to the documentation
|
||||
of this package for more information.
|
||||
|
||||
FileDownloadHandler
|
||||
-------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
|
||||
|
||||
| Supported scheme: ``file``.
|
||||
| Lazy: no.
|
||||
|
||||
This handler supports ``file:///path`` local file URIs. It doesn't
|
||||
support remote files.
|
||||
|
||||
FTPDownloadHandler
|
||||
------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
|
||||
|
||||
| Supported scheme: ``ftp``.
|
||||
| Lazy: no.
|
||||
|
||||
This handler supports ``ftp://host/path`` FTP URIs.
|
||||
|
||||
It's implemented using :mod:`twisted.protocols.ftp`.
|
||||
|
||||
.. note::
|
||||
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator
|
||||
|
||||
.. _twisted-http2-handler:
|
||||
|
||||
|
|
@ -177,7 +172,9 @@ H2DownloadHandler
|
|||
.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
|
||||
|
||||
| Supported scheme: ``https``.
|
||||
| Lazy: yes.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: yes.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
|
||||
|
||||
This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol
|
||||
for them.
|
||||
|
|
@ -196,63 +193,96 @@ If you want to use this handler you need to replace the default one for the
|
|||
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
|
||||
}
|
||||
|
||||
Features and limitations
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. warning::
|
||||
|
||||
This handler is experimental, and not yet recommended for production
|
||||
environments. Future Scrapy versions may introduce related changes without
|
||||
a deprecation period or warning.
|
||||
|
||||
.. note::
|
||||
=========================== ================================================
|
||||
HTTP proxies No (not implemented)
|
||||
SOCKS proxies No (not supported by the library)
|
||||
HTTP/2 Yes
|
||||
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
|
||||
Per-request ``bindaddress`` Yes
|
||||
TLS implementation ``pyOpenSSL``/``cryptography``
|
||||
=========================== ================================================
|
||||
|
||||
Known limitations of the HTTP/2 implementation in this handler include:
|
||||
Other limitations:
|
||||
|
||||
- No support for proxies.
|
||||
- No support for HTTP/1.1.
|
||||
|
||||
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
|
||||
HTTP/2 unencrypted (refer `http2 faq`_).
|
||||
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
|
||||
to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
|
||||
- No setting to specify a maximum `frame size`_ larger than the default
|
||||
value, 16384. Connections to servers that send a larger frame will
|
||||
fail.
|
||||
- No support for the :signal:`bytes_received` and :signal:`headers_received`
|
||||
signals.
|
||||
|
||||
- No support for `server pushes`_, which are ignored.
|
||||
Known limitations of the HTTP/2 support:
|
||||
|
||||
- No support for the :signal:`bytes_received` and
|
||||
:signal:`headers_received` signals.
|
||||
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
|
||||
HTTP/2 unencrypted (refer `http2 faq`_).
|
||||
|
||||
- No setting to specify a maximum `frame size`_ larger than the default
|
||||
value, 16384. Connections to servers that send a larger frame will fail.
|
||||
|
||||
- No support for `server pushes`_, which are ignored.
|
||||
|
||||
.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
|
||||
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
|
||||
.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
|
||||
|
||||
.. note::
|
||||
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
HTTP11DownloadHandler
|
||||
---------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler
|
||||
|
||||
| Supported schemes: ``http``, ``https``.
|
||||
| Lazy: no.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
|
||||
|
||||
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
|
||||
uses the HTTP/1.1 protocol for them.
|
||||
|
||||
It's implemented using :mod:`twisted.web.client`.
|
||||
|
||||
.. note::
|
||||
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
Features and limitations
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
=========================== ================================================
|
||||
HTTP proxies Yes
|
||||
SOCKS proxies No (not supported by the library)
|
||||
HTTP/2 No (implemented as a separate handler)
|
||||
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
|
||||
Per-request ``bindaddress`` Yes
|
||||
TLS implementation ``pyOpenSSL``/``cryptography``
|
||||
=========================== ================================================
|
||||
|
||||
Other limitations:
|
||||
|
||||
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
|
||||
to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
|
||||
- HTTPS proxies to HTTPS destinations are not supported.
|
||||
|
||||
HttpxDownloadHandler
|
||||
--------------------
|
||||
|
||||
.. versionadded:: 2.15.0
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
|
||||
|
||||
| Supported schemes: ``http``, ``https``.
|
||||
| Lazy: no.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: yes.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
|
||||
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
|
||||
uses the HTTP/1.1 protocol for them.
|
||||
uses the HTTP/1.1 or HTTP/2 protocol for them.
|
||||
|
||||
It's implemented using the ``httpx`` library and needs it to be installed.
|
||||
|
||||
|
|
@ -266,28 +296,83 @@ If you want to use this handler you need to replace the default ones for the
|
|||
"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
|
||||
}
|
||||
|
||||
Features and limitations
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. warning::
|
||||
|
||||
This handler is experimental, and not yet recommended for production
|
||||
environments. Future Scrapy versions may introduce related changes without
|
||||
a deprecation period or warning or even remove it altogether.
|
||||
|
||||
.. note::
|
||||
=========================== =======================================
|
||||
HTTP proxies Yes
|
||||
SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``)
|
||||
HTTP/2 Yes (requires ``httpx[http2]``)
|
||||
``response.certificate`` DER bytes
|
||||
Per-request ``bindaddress`` No (not supported by the library)
|
||||
TLS implementation Standard library ``ssl``
|
||||
=========================== =======================================
|
||||
|
||||
As this handler is based on a different HTTP client implementation compared
|
||||
to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on
|
||||
some websites may be different. Additionally, these are the Scrapy features
|
||||
that are explicitly not supported when using it:
|
||||
Other limitations:
|
||||
|
||||
- Per-request bind address support (the :reqmeta:`bindaddress` meta key).
|
||||
The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the
|
||||
port number, if specified, will be ignored.
|
||||
- The handler creates a separate connection pool for each proxy URL (due to
|
||||
limitations of ``httpx``) which may lead to higher resource usage when
|
||||
using proxy rotation.
|
||||
|
||||
- Settings specific to the Twisted networking or HTTP implementation, like
|
||||
:setting:`DNS_RESOLVER`.
|
||||
.. setting:: HTTPX_HTTP2_ENABLED
|
||||
|
||||
- Using :ref:`non-asyncio reactors <disable-asyncio>` (``httpx`` requires
|
||||
``asyncio``).
|
||||
HTTPX_HTTP2_ENABLED
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
|
||||
needs to be installed if you want to enable this setting.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Built-in non-HTTP download handlers reference
|
||||
=============================================
|
||||
|
||||
DataURIDownloadHandler
|
||||
----------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
|
||||
|
||||
| Supported scheme: ``data``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
|
||||
This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
|
||||
|
||||
FileDownloadHandler
|
||||
-------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
|
||||
|
||||
| Supported scheme: ``file``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
|
||||
This handler supports ``file:///path`` local file URIs. It doesn't
|
||||
support remote files.
|
||||
|
||||
FTPDownloadHandler
|
||||
------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
|
||||
|
||||
| Supported scheme: ``ftp``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
|
||||
|
||||
This handler supports ``ftp://host/path`` FTP URIs.
|
||||
|
||||
It's implemented using :mod:`twisted.protocols.ftp`.
|
||||
|
||||
S3DownloadHandler
|
||||
-----------------
|
||||
|
|
@ -295,7 +380,9 @@ S3DownloadHandler
|
|||
.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
|
||||
|
||||
| Supported scheme: ``s3``.
|
||||
| Lazy: yes.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: yes.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
|
||||
This handler supports ``s3://bucket/path`` S3 URIs.
|
||||
|
||||
|
|
|
|||
|
|
@ -745,8 +745,7 @@ HttpProxyMiddleware
|
|||
Handling of this meta key needs to be implemented inside the :ref:`download
|
||||
handler <topics-download-handlers>`, so it's not guaranteed to be supported
|
||||
by all 3rd-party handlers. It's currently unsupported by
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`.
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -758,6 +757,13 @@ HttpProxyMiddleware
|
|||
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
|
||||
supports HTTPS proxies only for HTTP destinations.
|
||||
|
||||
.. note::
|
||||
|
||||
If the download handler supports it, you can use a SOCKS proxy URL (e.g.
|
||||
``socks5://username:password@some_proxy_server:port``).
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`
|
||||
supports SOCKS proxies while other built-in handlers don't.
|
||||
|
||||
HttpProxyMiddleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||
import ipaddress
|
||||
import ssl
|
||||
from contextlib import asynccontextmanager
|
||||
from socket import gaierror
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from scrapy.exceptions import (
|
||||
|
|
@ -17,6 +18,7 @@ from scrapy.exceptions import (
|
|||
)
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils._download_handlers import NullCookieJar
|
||||
from scrapy.utils.python import _iter_exc_causes
|
||||
from scrapy.utils.ssl import (
|
||||
_log_sslobj_debug_info,
|
||||
_make_insecure_ssl_ctx,
|
||||
|
|
@ -34,10 +36,35 @@ if TYPE_CHECKING:
|
|||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
HAS_SOCKS = HAS_HTTP2 = False
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
httpx = None # type: ignore[assignment]
|
||||
else:
|
||||
# a small hack to avoid importing these optional extras unconditionally
|
||||
|
||||
DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = (
|
||||
httpx.RequestError,
|
||||
httpx.InvalidURL,
|
||||
)
|
||||
|
||||
try:
|
||||
import h2.exceptions
|
||||
|
||||
HAS_HTTP2 = True
|
||||
DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,)
|
||||
except ImportError: # pragma: no cover
|
||||
pass
|
||||
|
||||
try:
|
||||
import socksio.exceptions
|
||||
|
||||
HAS_SOCKS = True
|
||||
DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,)
|
||||
except ImportError: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -54,6 +81,11 @@ class HttpxDownloadHandler(_Base):
|
|||
self._verify_certificates: bool = crawler.settings.getbool(
|
||||
"DOWNLOAD_VERIFY_CERTIFICATES"
|
||||
)
|
||||
self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
|
||||
if self._enable_h2 and not HAS_HTTP2: # pragma: no cover
|
||||
raise NotConfigured(
|
||||
f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
|
||||
)
|
||||
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
|
||||
self._bind_host: str | None = self._get_bind_address_host()
|
||||
self._limits: httpx.Limits = httpx.Limits(
|
||||
|
|
@ -90,6 +122,7 @@ class HttpxDownloadHandler(_Base):
|
|||
transport=httpx.AsyncHTTPTransport(
|
||||
verify=self._ssl_context,
|
||||
local_address=self._bind_host,
|
||||
http2=self._enable_h2,
|
||||
limits=self._limits,
|
||||
trust_env=False,
|
||||
proxy=proxy,
|
||||
|
|
@ -113,7 +146,13 @@ class HttpxDownloadHandler(_Base):
|
|||
async def _make_request(
|
||||
self, request: Request, timeout: float
|
||||
) -> AsyncIterator[httpx.Response]:
|
||||
client = self._get_client(self._extract_proxy_url_with_creds(request))
|
||||
proxy = self._extract_proxy_url_with_creds(request)
|
||||
if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover
|
||||
raise ValueError(
|
||||
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
|
||||
)
|
||||
client = self._get_client(proxy)
|
||||
|
||||
try:
|
||||
async with client.stream(
|
||||
request.method,
|
||||
|
|
@ -130,18 +169,12 @@ class HttpxDownloadHandler(_Base):
|
|||
except httpx.UnsupportedProtocol as e:
|
||||
raise UnsupportedURLSchemeError(str(e)) from e
|
||||
except httpx.ConnectError as e:
|
||||
error_message = str(e)
|
||||
if (
|
||||
"Name or service not known" in error_message
|
||||
or "getaddrinfo failed" in error_message
|
||||
or "nodename nor servname" in error_message
|
||||
or "Temporary failure in name resolution" in error_message
|
||||
):
|
||||
raise CannotResolveHostError(error_message) from e
|
||||
if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)):
|
||||
raise CannotResolveHostError(str(e)) from e
|
||||
raise DownloadConnectionRefusedError(str(e)) from e
|
||||
except httpx.ProxyError as e:
|
||||
raise DownloadConnectionRefusedError(str(e)) from e
|
||||
except (httpx.NetworkError, httpx.RemoteProtocolError) as e:
|
||||
except DOWNLOAD_FAILED_EXCEPTIONS as e:
|
||||
raise DownloadFailedError(str(e)) from e
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ __all__ = [
|
|||
"HTTPCACHE_STORAGE",
|
||||
"HTTPPROXY_AUTH_ENCODING",
|
||||
"HTTPPROXY_ENABLED",
|
||||
"HTTPX_HTTP2_ENABLED",
|
||||
"IMAGES_STORE_GCS_ACL",
|
||||
"IMAGES_STORE_S3_ACL",
|
||||
"ITEM_PIPELINES",
|
||||
|
|
@ -382,6 +383,8 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
|||
HTTPPROXY_ENABLED = True
|
||||
HTTPPROXY_AUTH_ENCODING = "latin-1"
|
||||
|
||||
HTTPX_HTTP2_ENABLED = False
|
||||
|
||||
IMAGES_STORE_GCS_ACL = ""
|
||||
IMAGES_STORE_S3_ACL = "private"
|
||||
|
||||
|
|
|
|||
|
|
@ -359,3 +359,13 @@ def _looks_like_import_path(value: str) -> bool:
|
|||
if any(part == "" for part in parts):
|
||||
return False
|
||||
return all(part.isidentifier() for part in parts)
|
||||
|
||||
|
||||
def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]:
|
||||
"""Iterate over the exception causes/contexts."""
|
||||
seen: set[int] = set()
|
||||
cur: BaseException | None = exc
|
||||
while cur is not None and id(cur) not in seen:
|
||||
seen.add(id(cur))
|
||||
yield cur
|
||||
cur = cur.__cause__ or cur.__context__
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ class MitmProxy:
|
|||
auth_user = "scrapy"
|
||||
auth_pass = "scrapy"
|
||||
|
||||
def __init__(self, mode: str | None = None) -> None:
|
||||
self.mode = mode
|
||||
|
||||
def start(self) -> str:
|
||||
script = """
|
||||
import sys
|
||||
|
|
@ -32,6 +35,8 @@ sys.exit(mitmdump())
|
|||
"-s",
|
||||
str(Path(__file__).with_name("mitm_proxy_addon.py")),
|
||||
]
|
||||
if self.mode:
|
||||
args += ["--mode", self.mode]
|
||||
self.proc: Popen[str] = Popen(
|
||||
[
|
||||
sys.executable,
|
||||
|
|
@ -44,12 +49,13 @@ sys.exit(mitmdump())
|
|||
text=True,
|
||||
)
|
||||
assert self.proc.stdout is not None
|
||||
scheme = "socks5" if self.mode == "socks5" else "http"
|
||||
line = ""
|
||||
for line in self.proc.stdout:
|
||||
m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
|
||||
m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line)
|
||||
if m:
|
||||
host_port = m.group(1)
|
||||
return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
|
||||
return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}"
|
||||
self.stop()
|
||||
raise RuntimeError(f"Failed to parse mitmdump output: {line}")
|
||||
|
||||
|
|
|
|||
|
|
@ -3,11 +3,17 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.core.downloader.handlers._httpx import (
|
||||
HAS_HTTP2,
|
||||
HAS_SOCKS,
|
||||
HttpxDownloadHandler,
|
||||
)
|
||||
from scrapy.exceptions import DownloadFailedError
|
||||
from tests.test_downloader_handlers_http_base import (
|
||||
TestHttpBase,
|
||||
TestHttpProxyBase,
|
||||
|
|
@ -37,10 +43,6 @@ pytest.importorskip("httpx")
|
|||
class HttpxDownloadHandlerMixin:
|
||||
@property
|
||||
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
|
||||
from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415
|
||||
HttpxDownloadHandler,
|
||||
)
|
||||
|
||||
return HttpxDownloadHandler
|
||||
|
||||
@property
|
||||
|
|
@ -83,6 +85,30 @@ class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
|
|||
pass
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")
|
||||
class TestHttp2(TestHttps):
|
||||
http2 = True
|
||||
handler_supports_http2_dataloss = False
|
||||
|
||||
default_handler_settings: ClassVar[dict[str, Any]] = {
|
||||
"HTTPX_HTTP2_ENABLED": True,
|
||||
}
|
||||
|
||||
@coroutine_test
|
||||
async def test_protocol(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/host", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.protocol == "HTTP/2"
|
||||
|
||||
@coroutine_test
|
||||
async def test_data_loss_handling(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
with pytest.raises(DownloadFailedError):
|
||||
await download_handler.download_request(request)
|
||||
|
||||
|
||||
class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
|
||||
pass
|
||||
|
||||
|
|
@ -127,7 +153,7 @@ class TestHttpsProxy(TestHttpProxy):
|
|||
|
||||
@pytest.mark.requires_mitmproxy
|
||||
class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
|
||||
pass
|
||||
handler_supports_socks = HAS_SOCKS
|
||||
|
||||
|
||||
@pytest.mark.requires_internet
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ from scrapy.utils.misc import build_from_crawler
|
|||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests import NON_EXISTING_RESOLVABLE
|
||||
from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials
|
||||
from tests.mockserver.mitm_proxy import wrong_credentials
|
||||
from tests.mockserver.proxy_echo import ProxyEchoMockServer
|
||||
from tests.mockserver.simple_https import SimpleMockServer
|
||||
from tests.spiders import (
|
||||
|
|
@ -73,6 +73,7 @@ class TestHttpBase(ABC):
|
|||
handler_supports_http2_dataloss: bool = True
|
||||
# default headers added by the underlying library that cannot be suppressed
|
||||
always_present_req_headers: ClassVar[frozenset[str]] = frozenset()
|
||||
default_handler_settings: ClassVar[dict[str, Any]] = {}
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
|
|
@ -83,6 +84,10 @@ class TestHttpBase(ABC):
|
|||
async def get_dh(
|
||||
self, settings_dict: dict[str, Any] | None = None
|
||||
) -> AsyncGenerator[DownloadHandlerProtocol]:
|
||||
settings_dict = {
|
||||
**self.default_handler_settings,
|
||||
**(settings_dict or {}),
|
||||
}
|
||||
crawler = get_crawler(DefaultSpider, settings_dict)
|
||||
crawler.spider = crawler._create_spider()
|
||||
dh = build_from_crawler(self.download_handler_cls, crawler)
|
||||
|
|
@ -339,9 +344,7 @@ class TestHttpBase(ABC):
|
|||
|
||||
@coroutine_test
|
||||
async def test_timeout_download_from_spider_server_hangs(
|
||||
self,
|
||||
mockserver: MockServer,
|
||||
reactor_pytest: str,
|
||||
self, mockserver: MockServer, reactor_pytest: str
|
||||
) -> None:
|
||||
if reactor_pytest == "asyncio" and sys.platform == "win32":
|
||||
# https://twistedmatrix.com/trac/ticket/10279
|
||||
|
|
@ -1317,6 +1320,7 @@ class TestHttpProxyBase(ABC):
|
|||
class TestMitmProxyBase(ABC):
|
||||
# whether the handler supports HTTPS proxies with HTTPS destinations
|
||||
handler_supports_tls_in_tls: bool = True
|
||||
handler_supports_socks: bool = False
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
|
|
@ -1326,13 +1330,10 @@ class TestMitmProxyBase(ABC):
|
|||
@pytest.mark.parametrize(
|
||||
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
|
||||
)
|
||||
@pytest.mark.usefixtures("mitm_proxy_server")
|
||||
@coroutine_test
|
||||
async def test_http_proxy(
|
||||
self,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
mockserver: MockServer,
|
||||
mitm_proxy_server: MitmProxy,
|
||||
https_dest: bool,
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
|
||||
) -> None:
|
||||
"""HTTP proxy, HTTP or HTTPS destination."""
|
||||
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
|
||||
|
|
@ -1347,13 +1348,10 @@ class TestMitmProxyBase(ABC):
|
|||
@pytest.mark.parametrize(
|
||||
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
|
||||
)
|
||||
@pytest.mark.usefixtures("mitm_proxy_server_https")
|
||||
@coroutine_test
|
||||
async def test_https_proxy(
|
||||
self,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
mockserver: MockServer,
|
||||
mitm_proxy_server_https: MitmProxy,
|
||||
https_dest: bool,
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
|
||||
) -> None:
|
||||
"""HTTPS proxy, HTTP or HTTPS destination."""
|
||||
if https_dest and not self.handler_supports_tls_in_tls:
|
||||
|
|
@ -1370,13 +1368,13 @@ class TestMitmProxyBase(ABC):
|
|||
@pytest.mark.parametrize(
|
||||
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
|
||||
)
|
||||
@pytest.mark.usefixtures("mitm_proxy_server")
|
||||
@coroutine_test
|
||||
async def test_http_proxy_auth_error(
|
||||
self,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mockserver: MockServer,
|
||||
mitm_proxy_server: MitmProxy,
|
||||
https_dest: bool,
|
||||
) -> None:
|
||||
"""HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
|
||||
|
|
@ -1394,13 +1392,10 @@ class TestMitmProxyBase(ABC):
|
|||
@pytest.mark.parametrize(
|
||||
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
|
||||
)
|
||||
@pytest.mark.usefixtures("mitm_proxy_server")
|
||||
@coroutine_test
|
||||
async def test_dont_leak_proxy_authorization_header(
|
||||
self,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
mockserver: MockServer,
|
||||
mitm_proxy_server: MitmProxy,
|
||||
https_dest: bool,
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
|
||||
) -> None:
|
||||
"""HTTP proxy, HTTP or HTTPS destination. Check that the auth header
|
||||
is not sent to the destination."""
|
||||
|
|
@ -1414,6 +1409,49 @@ class TestMitmProxyBase(ABC):
|
|||
echo = json.loads(crawler.spider.meta["responses"][0].text)
|
||||
assert "Proxy-Authorization" not in echo["headers"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
|
||||
)
|
||||
@pytest.mark.usefixtures("socks5_proxy_server")
|
||||
@coroutine_test
|
||||
async def test_download_with_socks_proxy(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
|
||||
) -> None:
|
||||
"""SOCKS5 proxy, HTTP or HTTPS destination."""
|
||||
if not self.handler_supports_socks:
|
||||
pytest.skip("SOCKS proxies are not supported")
|
||||
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await crawler.crawl_async(
|
||||
seed=mockserver.url("/status?n=200", is_secure=https_dest)
|
||||
)
|
||||
assert isinstance(crawler.spider, SingleRequestSpider)
|
||||
self._assert_got_response_code(200, caplog.text)
|
||||
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
|
||||
)
|
||||
@pytest.mark.usefixtures("socks5_proxy_server")
|
||||
@coroutine_test
|
||||
async def test_socks_proxy_auth_error(
|
||||
self,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mockserver: MockServer,
|
||||
https_dest: bool,
|
||||
) -> None:
|
||||
if not self.handler_supports_socks:
|
||||
pytest.skip("SOCKS proxies are not supported")
|
||||
envvar = "https_proxy" if https_dest else "http_proxy"
|
||||
monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
|
||||
crawler = get_crawler(SimpleSpider, self.settings_dict)
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await crawler.crawl_async(
|
||||
mockserver.url("/status?n=200", is_secure=https_dest)
|
||||
)
|
||||
assert "DownloadConnectionRefusedError" in caplog.text
|
||||
|
||||
@staticmethod
|
||||
def _assert_headers(headers: Headers, https_dest: bool) -> None:
|
||||
assert b"X-Via-Mitmproxy" in headers
|
||||
|
|
|
|||
4
tox.ini
4
tox.ini
|
|
@ -60,6 +60,7 @@ deps =
|
|||
ipython==8.39.0
|
||||
pyOpenSSL==26.2.0
|
||||
pytest==9.0.3
|
||||
socksio==1.0.0
|
||||
types-Pygments==2.20.0.20260508
|
||||
types-defusedxml==0.7.0.20260504
|
||||
types-lxml==2026.2.16
|
||||
|
|
@ -148,7 +149,7 @@ deps =
|
|||
brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
|
||||
brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests
|
||||
google-cloud-storage
|
||||
httpx
|
||||
httpx[http2,socks]
|
||||
ipython
|
||||
robotexclusionrulesparser
|
||||
uvloop; platform_system != "Windows" and implementation_name != "pypy"
|
||||
|
|
@ -304,5 +305,6 @@ deps =
|
|||
{[testenv]deps}
|
||||
# mitmproxy does not support PyPy
|
||||
mitmproxy; implementation_name != "pypy"
|
||||
httpx[http2,socks]
|
||||
commands =
|
||||
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy
|
||||
|
|
|
|||
Loading…
Reference in New Issue