RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement HTTP Auth settings and request metadata keys (#7590) 

* Implement HTTP Auth settings and request metadata keys

* Fix doc-tests

* Require the domain to be set when using settings
This commit is contained in:
Adrian 2026-06-11 12:56:53 +02:00 committed by GitHub
parent c99f6e2209
commit a8ffdcf851
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 264 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
docs/topics/downloader-middleware.rst
View File

@ -307,26 +307,15 @@ HttpAuthMiddleware
.. class:: HttpAuthMiddleware
This middleware authenticates all requests generated from certain spiders
using `Basic access authentication`_ (aka. HTTP auth).
This middleware authenticates requests using `Basic access authentication`_
(aka. HTTP auth).
To enable HTTP authentication for a spider, set the ``http_user`` and
``http_pass`` spider attributes to the authentication data and the
``http_auth_domain`` spider attribute to the domain which requires this
authentication (its subdomains will be also handled in the same way).
You can set ``http_auth_domain`` to ``None`` to enable the
authentication for all requests but you risk leaking your authentication
credentials to unrelated domains.
Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and
:setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override
the credentials per request via :attr:`~scrapy.Request.meta` keys
:reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`.
.. warning::
In previous Scrapy versions HttpAuthMiddleware sent the authentication
data with all requests, which is a security problem if the spider
makes requests to several different domains. Currently if the
``http_auth_domain`` attribute is not set, the middleware will use the
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example:
Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`):
.. code-block:: python
@ -334,13 +323,62 @@ HttpAuthMiddleware
class SomeIntranetSiteSpider(CrawlSpider):
http_user = "someuser"
http_pass = "somepass"
http_auth_domain = "intranet.example.com"
name = "intranet.example.com"
custom_settings = {
"HTTPAUTH_USER": "someuser",
"HTTPAUTH_PASS": "somepass",
"HTTPAUTH_DOMAIN": "intranet.example.com",
}
# .. rest of the spider code omitted ...
Example using per-request meta:
.. code-block:: python
async def start(self):
yield Request(
"https://intranet.example.com/protected/",
meta={
"http_user": "someuser",
"http_pass": "somepass",
"http_auth_domain": "intranet.example.com",
},
)
.. setting:: HTTPAUTH_USER
HTTPAUTH_USER
~~~~~~~~~~~~~
Default: ``""``
The username to use for HTTP basic authentication, applied to all requests
whose URL matches :setting:`HTTPAUTH_DOMAIN`.
.. setting:: HTTPAUTH_PASS
HTTPAUTH_PASS
~~~~~~~~~~~~~
Default: ``""``
The password to use for HTTP basic authentication.
.. setting:: HTTPAUTH_DOMAIN
HTTPAUTH_DOMAIN
~~~~~~~~~~~~~~~
Default: ``None``
The domain (and its subdomains) to which HTTP basic authentication credentials
are sent. Set to ``None`` to send credentials with all requests, but be aware
that this risks leaking credentials to unrelated domains.
This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
or :setting:`HTTPAUTH_PASS` is set.
.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication

24
docs/topics/request-response.rst
View File

@ -724,6 +724,9 @@ Those are:
* :reqmeta:`give_up_log_level`
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`http_auth_domain`
* :reqmeta:`http_pass`
* :reqmeta:`http_user`
* :reqmeta:`is_start_request`
* :reqmeta:`max_retry_times`
* :reqmeta:`proxy`
@ -806,6 +809,27 @@ give_up_log_level
:ref:`Logging level <levels>` used for the message logged when a request
exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
.. reqmeta:: http_auth_domain
http_auth_domain
----------------
Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
.. reqmeta:: http_pass
http_pass
---------
Overrides :setting:`HTTPAUTH_PASS` for this request.
.. reqmeta:: http_user
http_user
---------
Overrides :setting:`HTTPAUTH_USER` for this request.
.. reqmeta:: max_retry_times
max_retry_times

5
docs/topics/spiders.rst
View File

@ -354,11 +354,6 @@ Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)
resulting in each character being seen as a separate url.
A valid use case is to set the http auth credentials
used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`::
scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword
Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
See `Scrapyd documentation`_.

55
scrapy/downloadermiddlewares/httpauth.py
View File

@ -6,11 +6,14 @@ See documentation in docs/topics/downloader-middleware.rst
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING
from w3lib.http import basic_auth_header
from scrapy import Request, Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import SETTINGS_PRIORITIES
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.url import url_is_from_any_domain
@ -23,12 +26,28 @@ if TYPE_CHECKING:
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header
(http_user and http_pass spider class attributes)"""
"""Set Basic HTTP Authorization header."""
def __init__(self) -> None:
self._auth: bytes | None = None
self._domain: str | None = None
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls()
usr = crawler.settings.get("HTTPAUTH_USER", "")
pwd = crawler.settings.get("HTTPAUTH_PASS", "")
if usr or pwd:
domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0
if domain_priority <= SETTINGS_PRIORITIES["default"]:
raise ValueError(
"HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS "
"is configured. Set it to a domain (e.g. 'example.com') to restrict "
"credentials to that domain, or set it to None to send credentials "
"with all requests."
)
o._auth = basic_auth_header(usr, pwd)
o._domain = crawler.settings.get("HTTPAUTH_DOMAIN")
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
@ -36,18 +55,34 @@ class HttpAuthMiddleware:
usr = getattr(spider, "http_user", "")
pwd = getattr(spider, "http_pass", "")
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
self.domain = spider.http_auth_domain # type: ignore[attr-defined]
warnings.warn(
"Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings "
"instead of the http_user, http_pass, and http_auth_domain spider "
"attributes. Support for the spider attributes will be removed in a "
"future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self._auth = basic_auth_header(usr, pwd)
self._domain = spider.http_auth_domain # type: ignore[attr-defined]
@_warn_spider_arg
def process_request(
self, request: Request, spider: Spider | None = None
) -> Request | Response | None:
auth = getattr(self, "auth", None)
if (
auth
and b"Authorization" not in request.headers
and (not self.domain or url_is_from_any_domain(request.url, [self.domain]))
if b"Authorization" in request.headers:
return None
# Per-request meta overrides
usr = request.meta.get("http_user", "")
pwd = request.meta.get("http_pass", "")
if usr or pwd:
domain = request.meta.get("http_auth_domain")
if not domain or url_is_from_any_domain(request.url, [domain]):
request.headers[b"Authorization"] = basic_auth_header(usr, pwd)
return None
# Middleware-level auth
if self._auth and (
not self._domain or url_is_from_any_domain(request.url, [self._domain])
):
request.headers[b"Authorization"] = auth
request.headers[b"Authorization"] = self._auth
return None

7
scrapy/settings/default_settings.py
View File

@ -107,6 +107,9 @@ __all__ = [
"FTP_PASSWORD",
"FTP_USER",
"GCS_PROJECT_ID",
"HTTPAUTH_DOMAIN",
"HTTPAUTH_PASS",
"HTTPAUTH_USER",
"HTTPCACHE_ALWAYS_STORE",
"HTTPCACHE_DBM_MODULE",
"HTTPCACHE_DIR",
@ -397,6 +400,10 @@ FTP_PASSWORD = "guest" # noqa: S105
GCS_PROJECT_ID = None
HTTPAUTH_USER = ""
HTTPAUTH_PASS = ""
HTTPAUTH_DOMAIN = None
HTTPCACHE_ENABLED = False
HTTPCACHE_ALWAYS_STORE = False
HTTPCACHE_DBM_MODULE = "dbm"

167
tests/test_downloadermiddleware_httpauth.py
View File

@ -2,8 +2,25 @@ import pytest
from w3lib.http import basic_auth_header
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
_DOMAIN_NOT_SET = object()
def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
settings: dict = {
"HTTPAUTH_USER": user,
"HTTPAUTH_PASS": passwd,
}
if domain is not _DOMAIN_NOT_SET:
settings["HTTPAUTH_DOMAIN"] = domain
return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings))
# --- Spider attribute tests (deprecated) ---
class LegacySpider(Spider):
@ -23,61 +40,135 @@ class AnyDomainSpider(Spider):
http_auth_domain = None
class TestHttpAuthMiddlewareLegacy:
def setup_method(self):
self.spider = LegacySpider("foo")
def test_auth(self):
class TestHttpAuthMiddlewareLegacySpiderAttr:
def test_missing_domain_raises(self):
mw = HttpAuthMiddleware()
with pytest.raises(AttributeError):
mw.spider_opened(self.spider)
with pytest.warns(ScrapyDeprecationWarning), pytest.raises(AttributeError):
mw.spider_opened(LegacySpider("foo"))
def test_domain_spider(self):
mw = HttpAuthMiddleware()
with pytest.warns(ScrapyDeprecationWarning):
mw.spider_opened(DomainSpider("foo"))
req = Request("http://example.com/")
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
class TestHttpAuthMiddleware:
def setup_method(self):
self.mw = HttpAuthMiddleware()
spider = DomainSpider("foo")
self.mw.spider_opened(spider)
def teardown_method(self):
del self.mw
def test_no_auth(self):
req = Request("http://example-noauth.com/")
assert self.mw.process_request(req) is None
def test_no_auth_wrong_domain(self):
mw = HttpAuthMiddleware()
with pytest.warns(ScrapyDeprecationWarning):
mw.spider_opened(DomainSpider("foo"))
req = Request("http://other.com/")
mw.process_request(req)
assert "Authorization" not in req.headers
def test_auth_domain(self):
def test_any_domain_spider(self):
mw = HttpAuthMiddleware()
with pytest.warns(ScrapyDeprecationWarning):
mw.spider_opened(AnyDomainSpider("foo"))
req = Request("http://anywhere.com/")
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
# --- Settings-based tests ---
class TestHttpAuthMiddlewareSettings:
def test_no_auth(self):
mw = make_mw()
req = Request("http://example.com/")
assert self.mw.process_request(req) is None
mw.process_request(req)
assert "Authorization" not in req.headers
def test_auth_without_domain_raises(self):
with pytest.raises(ValueError, match="HTTPAUTH_DOMAIN"):
make_mw(user="foo", passwd="bar")
def test_auth_all_domains(self):
mw = make_mw(user="foo", passwd="bar", domain=None)
req = Request("http://example.com/")
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_domain_match(self):
mw = make_mw(user="foo", passwd="bar", domain="example.com")
req = Request("http://example.com/")
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_auth_subdomain(self):
req = Request("http://foo.example.com/")
assert self.mw.process_request(req) is None
mw = make_mw(user="foo", passwd="bar", domain="example.com")
req = Request("http://sub.example.com/")
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
def test_no_auth_wrong_domain(self):
mw = make_mw(user="foo", passwd="bar", domain="example.com")
req = Request("http://other.com/")
mw.process_request(req)
assert "Authorization" not in req.headers
def test_auth_already_set(self):
mw = make_mw(user="foo", passwd="bar", domain="example.com")
req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req) is None
mw.process_request(req)
assert req.headers["Authorization"] == b"Digest 123"
class TestHttpAuthAnyMiddleware:
def setup_method(self):
self.mw = HttpAuthMiddleware()
spider = AnyDomainSpider("foo")
self.mw.spider_opened(spider)
# --- Per-request meta tests ---
def teardown_method(self):
del self.mw
def test_auth(self):
req = Request("http://example.com/")
assert self.mw.process_request(req) is None
assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
class TestHttpAuthMiddlewareMeta:
def test_meta_auth_no_domain(self):
mw = make_mw()
req = Request("http://example.com/", meta={"http_user": "u", "http_pass": "p"})
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("u", "p")
def test_auth_already_set(self):
req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req) is None
def test_meta_auth_domain_match(self):
mw = make_mw()
req = Request(
"http://example.com/",
meta={
"http_user": "u",
"http_pass": "p",
"http_auth_domain": "example.com",
},
)
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header("u", "p")
def test_meta_auth_domain_no_match(self):
mw = make_mw()
req = Request(
"http://other.com/",
meta={
"http_user": "u",
"http_pass": "p",
"http_auth_domain": "example.com",
},
)
mw.process_request(req)
assert "Authorization" not in req.headers
def test_meta_overrides_middleware(self):
mw = make_mw(user="mw_user", passwd="mw_pass", domain="example.com")
req = Request(
"http://example.com/",
meta={"http_user": "meta_user", "http_pass": "meta_pass"},
)
mw.process_request(req)
assert req.headers["Authorization"] == basic_auth_header(
"meta_user", "meta_pass"
)
def test_meta_already_set(self):
mw = make_mw()
req = Request(
"http://example.com/",
headers={"Authorization": "Digest 123"},
meta={"http_user": "u", "http_pass": "p"},
)
mw.process_request(req)
assert req.headers["Authorization"] == b"Digest 123"