RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup/clarifications of proxy support and improvements for proxy tests (#7496) 

* Run mitmproxy-based tests for every handler and improve them.

* Fixes for H2DownloadHandler.

* Revert "Fixes for H2DownloadHandler."

This reverts commit bcdbd097cd.

* Add TestHttpsProxy for HTTP11DownloadHandler.

* Use the configured context factory in ScrapyProxyAgent.

* Reword.

* Reword.

* pragma: no cover for H2DownloadHandler proxy code.

* Raise an exception for HTTPS proxies instead of using HTTP.

* Remove non-working proxy support and explicitly forbid HTTP.

* pragma: no cover

* Add a docs note about HTTPS proxies.

* Rename ScrapyProxyAgent.
This commit is contained in:
Andrey Rakhmatullin 2026-05-12 17:45:54 +05:00 committed by GitHub
parent 7f15ca92fc
commit 7fc84d372a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 323 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
conftest.py
View File

@ -11,6 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
from scrapy.utils.reactorless import install_reactor_import_hook
from tests.keys import generate_keys
from tests.mockserver.http import MockServer
from tests.mockserver.mitm_proxy import MitmProxy
if TYPE_CHECKING:
from collections.abc import Generator
@ -72,6 +73,32 @@ def mockserver() -> Generator[MockServer]:
yield mockserver
@pytest.fixture # function scope because it modifies os.environ
def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
proxy = MitmProxy()
url = proxy.start()
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
yield proxy
finally:
proxy.stop()
@pytest.fixture # function scope because it modifies os.environ
def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
proxy = MitmProxy()
url = proxy.start().replace("http://", "https://")
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
yield proxy
finally:
proxy.stop()
@pytest.fixture(scope="session")
def reactor_pytest(request) -> str:
return request.config.getoption("--reactor")

11
docs/news.rst
View File

@ -3,6 +3,17 @@
Release notes
=============
Scrapy VERSION (unreleased)
---------------------------
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made
private as it's an implementation detail of
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
(:issue:`7496`)
.. _release-2.15.2:
Scrapy 2.15.2 (2026-04-28)

2
docs/topics/download-handlers.rst
View File

@ -206,6 +206,8 @@ If you want to use this handler you need to replace the default one for the
Known limitations of the HTTP/2 implementation in this handler include:
- No support for proxies.
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).

11
docs/topics/downloader-middleware.rst
View File

@ -745,8 +745,19 @@ HttpProxyMiddleware
Handling of this meta key needs to be implemented inside the :ref:`download
handler <topics-download-handlers>`, so it's not guaranteed to be supported
by all 3rd-party handlers. It's currently unsupported by
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
.. note::
Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the
``https://`` one. While both kinds of proxy URLs can be used with both HTTP
and HTTPS destination URLs, the specifics of the network exchange are
different for all 4 cases and it's possible that HTTPS proxies are fully or
partially unsupported by a given download handler. Currently,
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
supports HTTPS proxies only for HTTP destinations.
HttpProxyMiddleware settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

12
scrapy/core/downloader/handlers/http11.py
View File

@ -338,17 +338,19 @@ class TunnelingAgent(Agent):
)
class ScrapyProxyAgent(Agent):
class _ScrapyProxyAgent(Agent):
def __init__(
self,
reactor: ReactorBase,
proxyURI: bytes,
contextFactory: IPolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
):
super().__init__( # type: ignore[no-untyped-call]
reactor=reactor,
contextFactory=contextFactory,
connectTimeout=connectTimeout,
bindAddress=bindAddress,
pool=pool,
@ -380,7 +382,6 @@ class ScrapyProxyAgent(Agent):
class ScrapyAgent:
_Agent = Agent
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(
@ -421,6 +422,10 @@ class ScrapyAgent:
if not proxy_port:
proxy_port = 443 if proxy_parsed.scheme == "https" else 80
if urlparse_cached(request).scheme == "https":
if proxy_parsed.scheme == "https": # pragma: no cover
raise NotImplementedError(
"HTTPS proxies for HTTPS destinations are not supported"
)
assert proxy_host is not None
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxy_host, proxy_port, proxyAuth)
@ -432,9 +437,10 @@ class ScrapyAgent:
bindAddress=bindaddress,
pool=self._pool,
)
return self._ProxyAgent(
return _ScrapyProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxy, encoding="ascii"),
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,

34
scrapy/core/downloader/handlers/http2.py
View File

@ -4,19 +4,20 @@ from time import monotonic
from typing import TYPE_CHECKING
from urllib.parse import urldefrag
from twisted.web.client import URI
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
from scrapy.exceptions import DownloadTimeoutError, NotConfigured
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
from scrapy.exceptions import (
DownloadTimeoutError,
NotConfigured,
UnsupportedURLSchemeError,
)
from scrapy.utils._download_handlers import (
normalize_bind_address,
wrap_twisted_exceptions,
)
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from twisted.internet.base import DelayedCall
@ -44,6 +45,10 @@ class H2DownloadHandler(BaseDownloadHandler):
self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
async def download_request(self, request: Request) -> Response:
if urlparse_cached(request).scheme == "http": # pragma: no cover
raise UnsupportedURLSchemeError(
f"{type(self).__name__} doesn't support plain HTTP."
)
agent = ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
@ -62,7 +67,6 @@ class H2DownloadHandler(BaseDownloadHandler):
class ScrapyH2Agent:
_Agent = H2Agent
_ProxyAgent = ScrapyProxyH2Agent
def __init__(
self,
@ -81,24 +85,10 @@ class ScrapyH2Agent:
def _get_agent(self, request: Request, timeout: float | None) -> H2Agent:
from twisted.internet import reactor
if request.meta.get("proxy"): # pragma: no cover
raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")
bind_address = request.meta.get("bindaddress") or self._bind_address
bind_address = normalize_bind_address(bind_address)
proxy = request.meta.get("proxy")
if proxy:
if urlparse_cached(request).scheme == "https":
# ToDo
raise NotImplementedError(
"Tunneling via CONNECT method using HTTP/2.0 is not yet supported"
)
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
)
return self._Agent(
reactor=reactor,
context_factory=self._context_factory,

27
scrapy/core/http2/agent.py
View File

@ -165,30 +165,3 @@ class H2Agent:
lambda conn: conn.request(request, spider)
)
return d2
class ScrapyProxyH2Agent(H2Agent):
def __init__(
self,
reactor: ReactorBase,
proxy_uri: URI,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008
connect_timeout: float | None = None,
bind_address: tuple[str, int] | None = None,
) -> None:
super().__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,
connect_timeout=connect_timeout,
bind_address=bind_address,
)
self._proxy_uri = proxy_uri
def get_endpoint(self, uri: URI) -> HostnameEndpoint:
return self.endpoint_factory.endpointForURI(self._proxy_uri) # type: ignore[no-any-return]
def get_key(self, uri: URI) -> ConnectionKeyT:
"""We use the proxy uri instead of uri obtained from request url"""
return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port

64
tests/mockserver/mitm_proxy.py Normal file
View File

@ -0,0 +1,64 @@
from __future__ import annotations
import re
import sys
from pathlib import Path
from subprocess import PIPE, Popen
from urllib.parse import urlsplit, urlunsplit
class MitmProxy:
auth_user = "scrapy"
auth_pass = "scrapy"
def start(self) -> str:
script = """
import sys
from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
cert_path = Path(__file__).parent.parent.resolve() / "keys"
args = [
"--listen-host",
"127.0.0.1",
"--listen-port",
"0",
"--proxyauth",
f"{self.auth_user}:{self.auth_pass}",
"--set",
f"confdir={cert_path}",
"--ssl-insecure",
"-s",
str(Path(__file__).with_name("mitm_proxy_addon.py")),
]
self.proc: Popen[str] = Popen(
[
sys.executable,
"-u",
"-c",
script,
*args,
],
stdout=PIPE,
text=True,
)
assert self.proc.stdout is not None
line = ""
for line in self.proc.stdout:
m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
if m:
host_port = m.group(1)
return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
self.stop()
raise RuntimeError(f"Failed to parse mitmdump output: {line}")
def stop(self) -> None:
self.proc.kill()
self.proc.communicate()
def wrong_credentials(proxy_url: str) -> str:
bad_auth_proxy = list(urlsplit(proxy_url))
bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")
return urlunsplit(bad_auth_proxy)

5
tests/mockserver/mitm_proxy_addon.py Normal file
View File

@ -0,0 +1,5 @@
def response(flow) -> None:
# add custom headers to be able to check that the request went through the proxy
flow.response.headers["X-Via-Mitmproxy"] = "1"
if flow.client_conn.tls_established:
flow.response.headers["X-Via-Mitmproxy-TLS"] = "1"

27
tests/test_downloader_handler_httpx.py
View File

@ -17,6 +17,7 @@ from tests.test_downloader_handlers_http_base import (
TestHttpsInvalidDNSPatternBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test
@ -41,6 +42,15 @@ class HttpxDownloadHandlerMixin:
return HttpxDownloadHandler
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
}
}
class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
handler_supports_bindaddress_meta = False
@ -108,15 +118,8 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBa
pass
class TestHttpWithCrawler(TestHttpWithCrawlerBase):
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
}
}
class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):
pass
class TestHttpsWithCrawler(TestHttpWithCrawler):
@ -136,3 +139,9 @@ class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
@pytest.mark.skip(reason="Proxy support is not implemented yet")
class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
is_secure = True
@pytest.mark.skip(reason="Proxy support is not implemented yet")
@pytest.mark.requires_mitmproxy
class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
pass

33
tests/test_downloader_handler_twisted_http11.py
View File

@ -19,6 +19,7 @@ from tests.test_downloader_handlers_http_base import (
TestHttpsInvalidDNSPatternBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestSimpleHttpsBase,
)
@ -34,6 +35,15 @@ class HTTP11DownloadHandlerMixin:
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return HTTP11DownloadHandler
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
"https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
}
}
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
@ -71,15 +81,8 @@ class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersB
pass
class TestHttpWithCrawler(TestHttpWithCrawlerBase):
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
"https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
}
}
class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase):
pass
class TestHttpsWithCrawler(TestHttpWithCrawler):
@ -88,3 +91,15 @@ class TestHttpsWithCrawler(TestHttpWithCrawler):
class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
pass
class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
is_secure = True
# not implemented
handler_supports_tls_in_tls = False
@pytest.mark.requires_mitmproxy
class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):
# not implemented
handler_supports_tls_in_tls = False

51
tests/test_downloader_handler_twisted_http2.py
View File

@ -10,13 +10,8 @@ from twisted.web.http import H2_ENABLED
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import (
DownloadFailedError,
NotConfigured,
UnsupportedURLSchemeError,
)
from scrapy.exceptions import DownloadFailedError, NotConfigured
from scrapy.http import Request
from scrapy.utils.defer import maybe_deferred_to_future
from tests.test_downloader_handlers_http_base import (
TestHttpProxyBase,
TestHttpsBase,
@ -25,13 +20,13 @@ from tests.test_downloader_handlers_http_base import (
TestHttpsInvalidDNSPatternBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
from tests.mockserver.http import MockServer
from tests.mockserver.proxy_echo import ProxyEchoMockServer
pytestmark = [
@ -52,6 +47,15 @@ class H2DownloadHandlerMixin:
return H2DownloadHandler
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": None,
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
}
}
def test_not_configured_without_reactor() -> None:
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler # noqa: PLC0415
@ -167,16 +171,7 @@ class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase)
pass
class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": None,
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
}
}
class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase):
is_secure = True
def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override]
@ -192,24 +187,12 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
pytest.skip("headers_received support is not implemented")
@pytest.mark.skip(reason="Proxy support is not implemented yet")
class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
is_secure = True
expected_http_proxy_request_body = b"/"
@coroutine_test
async def test_download_with_proxy_https_timeout(
self, proxy_mockserver: ProxyEchoMockServer
) -> None:
with pytest.raises(NotImplementedError):
await maybe_deferred_to_future(
super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type]
)
@coroutine_test
async def test_download_with_proxy_without_http_scheme(
self, proxy_mockserver: ProxyEchoMockServer
) -> None:
with pytest.raises(UnsupportedURLSchemeError):
await maybe_deferred_to_future(
super().test_download_with_proxy_without_http_scheme(proxy_mockserver) # type: ignore[arg-type]
)
@pytest.mark.skip(reason="Proxy support is not implemented yet")
@pytest.mark.requires_mitmproxy
class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase):
pass

123
tests/test_downloader_handlers_http_base.py
View File

@ -4,6 +4,8 @@ from __future__ import annotations
import gzip
import json
import logging
import os
import platform
import re
import sys
@ -36,6 +38,7 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
from tests.spiders import (
@ -43,6 +46,7 @@ from tests.spiders import (
BytesReceivedErrbackSpider,
HeadersReceivedCallbackSpider,
HeadersReceivedErrbackSpider,
SimpleSpider,
SingleRequestSpider,
)
from tests.utils.decorators import coroutine_test
@ -1072,6 +1076,8 @@ class TestHttpWithCrawlerBase(ABC):
class TestHttpProxyBase(ABC):
is_secure = False
expected_http_proxy_request_body = b"http://example.com"
# whether the handler supports HTTPS proxies with HTTPS destinations
handler_supports_tls_in_tls: bool = True
@property
@abstractmethod
@ -1124,6 +1130,8 @@ class TestHttpProxyBase(ABC):
) -> None:
if NON_EXISTING_RESOLVABLE:
pytest.skip("Non-existing hosts are resolvable")
if self.is_secure and not self.handler_supports_tls_in_tls:
pytest.skip("HTTPS proxies for HTTPS destinations are not supported")
http_proxy = proxy_mockserver.url("", is_secure=self.is_secure)
domain = "https://no-such-domain.nosuch"
request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})
@ -1143,3 +1151,118 @@ class TestHttpProxyBase(ABC):
assert response.status == 200
assert response.url == request.url
assert response.body == self.expected_http_proxy_request_body
class TestMitmProxyBase(ABC):
# whether the handler supports HTTPS proxies with HTTPS destinations
handler_supports_tls_in_tls: bool = True
@property
@abstractmethod
def settings_dict(self) -> dict[str, Any] | None:
raise NotImplementedError
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@coroutine_test
async def test_http_proxy(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
mitm_proxy_server: MitmProxy,
https_dest: bool,
) -> None:
"""HTTP proxy, HTTP or HTTPS destination."""
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@coroutine_test
async def test_https_proxy(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
mitm_proxy_server_https: MitmProxy,
https_dest: bool,
) -> None:
"""HTTPS proxy, HTTP or HTTPS destination."""
if https_dest and not self.handler_supports_tls_in_tls:
pytest.skip("HTTPS proxies for HTTPS destinations are not supported")
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@coroutine_test
async def test_http_proxy_auth_error(
self,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
mockserver: MockServer,
mitm_proxy_server: MitmProxy,
https_dest: bool,
) -> None:
"""HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
envvar = "https_proxy" if https_dest else "http_proxy"
monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
crawler = get_crawler(SimpleSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
mockserver.url("/status?n=200", is_secure=https_dest)
)
# The proxy returns a 407 error code but it does not reach the client;
# it just sees an exception.
self._assert_got_auth_exception(caplog.text)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@coroutine_test
async def test_dont_leak_proxy_authorization_header(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
mitm_proxy_server: MitmProxy,
https_dest: bool,
) -> None:
"""HTTP proxy, HTTP or HTTPS destination. Check that the auth header
is not sent to the destination."""
request = Request(mockserver.url("/echo", is_secure=https_dest))
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(seed=request)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
echo = json.loads(crawler.spider.meta["responses"][0].text)
assert "Proxy-Authorization" not in echo["headers"]
@staticmethod
def _assert_headers(headers: Headers, https_dest: bool) -> None:
assert b"X-Via-Mitmproxy" in headers
if https_dest:
assert b"X-Via-Mitmproxy-TLS" in headers
@staticmethod
def _assert_got_response_code(code: int, log: str) -> None:
assert str(log).count(f"Crawled ({code})") == 1
@staticmethod
def _assert_got_auth_exception(log: str) -> None:
assert "Proxy Authentication Required" in log or "407" in log

125
tests/test_proxy_connect.py
View File

@ -1,125 +0,0 @@
import json
import os
import re
import sys
from pathlib import Path
from subprocess import PIPE, Popen
from urllib.parse import urlsplit, urlunsplit
import pytest
from testfixtures import LogCapture
from scrapy.http import Request
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import SimpleSpider, SingleRequestSpider
from tests.utils.decorators import inline_callbacks_test
class MitmProxy:
auth_user = "scrapy"
auth_pass = "scrapy"
def start(self) -> str:
script = """
import sys
from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
cert_path = Path(__file__).parent.resolve() / "keys"
args = [
"--listen-host",
"127.0.0.1",
"--listen-port",
"0",
"--proxyauth",
f"{self.auth_user}:{self.auth_pass}",
"--set",
f"confdir={cert_path}",
"--ssl-insecure",
]
self.proc = Popen(
[
sys.executable,
"-u",
"-c",
script,
*args,
],
stdout=PIPE,
text=True,
)
assert self.proc.stdout is not None
line = self.proc.stdout.readline()
m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
if not m:
raise RuntimeError(f"Failed to parse mitmdump output: {line}")
host_port = m.group(1)
return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
def stop(self) -> None:
self.proc.kill()
self.proc.communicate()
def _wrong_credentials(proxy_url: str) -> str:
bad_auth_proxy = list(urlsplit(proxy_url))
bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")
return urlunsplit(bad_auth_proxy)
@pytest.mark.requires_mitmproxy
class TestProxyConnect:
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
def setup_method(self):
self._oldenv = os.environ.copy()
self._proxy = MitmProxy()
proxy_url = self._proxy.start()
os.environ["https_proxy"] = proxy_url
os.environ["http_proxy"] = proxy_url
def teardown_method(self):
self._proxy.stop()
os.environ = self._oldenv
@inline_callbacks_test
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, log)
@inline_callbacks_test
def test_https_tunnel_auth_error(self):
os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
# The proxy returns a 407 error code but it does not reach the client;
# he just sees a TunnelError.
self._assert_got_tunnel_error(log)
@inline_callbacks_test
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)
with LogCapture() as log:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, log)
echo = json.loads(crawler.spider.meta["responses"][0].text)
assert "Proxy-Authorization" not in echo["headers"]
def _assert_got_response_code(self, code, log):
assert str(log).count(f"Crawled ({code})") == 1
def _assert_got_tunnel_error(self, log):
assert "TunnelError" in str(log)