Implement HTTP Auth settings and request metadata keys (#7590)

* Implement HTTP Auth settings and request metadata keys

* Fix doc-tests

* Require the domain to be set when using settings

docs/topics/downloader-middleware.rst

@@ -307,26 +307,15 @@ HttpAuthMiddleware
 
 .. class:: HttpAuthMiddleware
 
-    This middleware authenticates all requests generated from certain spiders
+    This middleware authenticates requests using `Basic access authentication`_
-    using `Basic access authentication`_ (aka. HTTP auth).
+    (aka. HTTP auth).
 
-    To enable HTTP authentication for a spider, set the ``http_user`` and
+    Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and
-    ``http_pass`` spider attributes to the authentication data and the
+    :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override
-    ``http_auth_domain`` spider attribute to the domain which requires this
+    the credentials per request via :attr:`~scrapy.Request.meta` keys
-    authentication (its subdomains will be also handled in the same way).
+    :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`.
-    You can set ``http_auth_domain`` to ``None`` to enable the
-    authentication for all requests but you risk leaking your authentication
-    credentials to unrelated domains.
 
-    .. warning::
+    Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`):
-        In previous Scrapy versions HttpAuthMiddleware sent the authentication
-        data with all requests, which is a security problem if the spider
-        makes requests to several different domains. Currently if the
-        ``http_auth_domain`` attribute is not set, the middleware will use the
-        domain of the first request, which will work for some spiders but not
-        for others. In the future the middleware will produce an error instead.
-
-    Example:
 
     .. code-block:: python
 
@@ -334,13 +323,62 @@ HttpAuthMiddleware
 
 
         class SomeIntranetSiteSpider(CrawlSpider):
-            http_user = "someuser"
-            http_pass = "somepass"
-            http_auth_domain = "intranet.example.com"
             name = "intranet.example.com"
+            custom_settings = {
+                "HTTPAUTH_USER": "someuser",
+                "HTTPAUTH_PASS": "somepass",
+                "HTTPAUTH_DOMAIN": "intranet.example.com",
+            }
 
             # .. rest of the spider code omitted ...
 
+    Example using per-request meta:
+
+    .. code-block:: python
+
+        async def start(self):
+            yield Request(
+                "https://intranet.example.com/protected/",
+                meta={
+                    "http_user": "someuser",
+                    "http_pass": "somepass",
+                    "http_auth_domain": "intranet.example.com",
+                },
+            )
+
+.. setting:: HTTPAUTH_USER
+
+HTTPAUTH_USER
+~~~~~~~~~~~~~
+
+Default: ``""``
+
+The username to use for HTTP basic authentication, applied to all requests
+whose URL matches :setting:`HTTPAUTH_DOMAIN`.
+
+.. setting:: HTTPAUTH_PASS
+
+HTTPAUTH_PASS
+~~~~~~~~~~~~~
+
+Default: ``""``
+
+The password to use for HTTP basic authentication.
+
+.. setting:: HTTPAUTH_DOMAIN
+
+HTTPAUTH_DOMAIN
+~~~~~~~~~~~~~~~
+
+Default: ``None``
+
+The domain (and its subdomains) to which HTTP basic authentication credentials
+are sent. Set to ``None`` to send credentials with all requests, but be aware
+that this risks leaking credentials to unrelated domains.
+
+This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
+or :setting:`HTTPAUTH_PASS` is set.
+
 .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
 
 

docs/topics/request-response.rst

@@ -724,6 +724,9 @@ Those are:
 * :reqmeta:`give_up_log_level`
 * :reqmeta:`handle_httpstatus_all`
 * :reqmeta:`handle_httpstatus_list`
+* :reqmeta:`http_auth_domain`
+* :reqmeta:`http_pass`
+* :reqmeta:`http_user`
 * :reqmeta:`is_start_request`
 * :reqmeta:`max_retry_times`
 * :reqmeta:`proxy`
@@ -806,6 +809,27 @@ give_up_log_level
 :ref:`Logging level <levels>` used for the message logged when a request
 exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
 
+.. reqmeta:: http_auth_domain
+
+http_auth_domain
+----------------
+
+Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
+
+.. reqmeta:: http_pass
+
+http_pass
+---------
+
+Overrides :setting:`HTTPAUTH_PASS` for this request.
+
+.. reqmeta:: http_user
+
+http_user
+---------
+
+Overrides :setting:`HTTPAUTH_USER` for this request.
+
 .. reqmeta:: max_retry_times
 
 max_retry_times

docs/topics/spiders.rst

@@ -354,11 +354,6 @@ Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
 resulting in each character being seen as a separate url.
 
-A valid use case is to set the http auth credentials
-used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`::
-
-    scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword
-
 Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
 See `Scrapyd documentation`_.
 

scrapy/downloadermiddlewares/httpauth.py

@@ -6,11 +6,14 @@ See documentation in docs/topics/downloader-middleware.rst
 
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING
 
 from w3lib.http import basic_auth_header
 
 from scrapy import Request, Spider, signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings import SETTINGS_PRIORITIES
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.url import url_is_from_any_domain
 
@@ -23,12 +26,28 @@ if TYPE_CHECKING:
 
 
 class HttpAuthMiddleware:
-    """Set Basic HTTP Authorization header
+    """Set Basic HTTP Authorization header."""
-    (http_user and http_pass spider class attributes)"""
+
+    def __init__(self) -> None:
+        self._auth: bytes | None = None
+        self._domain: str | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         o = cls()
+        usr = crawler.settings.get("HTTPAUTH_USER", "")
+        pwd = crawler.settings.get("HTTPAUTH_PASS", "")
+        if usr or pwd:
+            domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0
+            if domain_priority <= SETTINGS_PRIORITIES["default"]:
+                raise ValueError(
+                    "HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS "
+                    "is configured. Set it to a domain (e.g. 'example.com') to restrict "
+                    "credentials to that domain, or set it to None to send credentials "
+                    "with all requests."
+                )
+            o._auth = basic_auth_header(usr, pwd)
+            o._domain = crawler.settings.get("HTTPAUTH_DOMAIN")
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         return o
 
@@ -36,18 +55,34 @@ class HttpAuthMiddleware:
         usr = getattr(spider, "http_user", "")
         pwd = getattr(spider, "http_pass", "")
         if usr or pwd:
-            self.auth = basic_auth_header(usr, pwd)
+            warnings.warn(
-            self.domain = spider.http_auth_domain  # type: ignore[attr-defined]
+                "Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings "
+                "instead of the http_user, http_pass, and http_auth_domain spider "
+                "attributes. Support for the spider attributes will be removed in a "
+                "future version of Scrapy.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self._auth = basic_auth_header(usr, pwd)
+            self._domain = spider.http_auth_domain  # type: ignore[attr-defined]
 
     @_warn_spider_arg
     def process_request(
         self, request: Request, spider: Spider | None = None
     ) -> Request | Response | None:
-        auth = getattr(self, "auth", None)
+        if b"Authorization" in request.headers:
-        if (
+            return None
-            auth
+        # Per-request meta overrides
-            and b"Authorization" not in request.headers
+        usr = request.meta.get("http_user", "")
-            and (not self.domain or url_is_from_any_domain(request.url, [self.domain]))
+        pwd = request.meta.get("http_pass", "")
+        if usr or pwd:
+            domain = request.meta.get("http_auth_domain")
+            if not domain or url_is_from_any_domain(request.url, [domain]):
+                request.headers[b"Authorization"] = basic_auth_header(usr, pwd)
+            return None
+        # Middleware-level auth
+        if self._auth and (
+            not self._domain or url_is_from_any_domain(request.url, [self._domain])
         ):
-            request.headers[b"Authorization"] = auth
+            request.headers[b"Authorization"] = self._auth
         return None

scrapy/settings/default_settings.py

@@ -107,6 +107,9 @@ __all__ = [
     "FTP_PASSWORD",
     "FTP_USER",
     "GCS_PROJECT_ID",
+    "HTTPAUTH_DOMAIN",
+    "HTTPAUTH_PASS",
+    "HTTPAUTH_USER",
     "HTTPCACHE_ALWAYS_STORE",
     "HTTPCACHE_DBM_MODULE",
     "HTTPCACHE_DIR",
@@ -397,6 +400,10 @@ FTP_PASSWORD = "guest"  # noqa: S105
 
 GCS_PROJECT_ID = None
 
+HTTPAUTH_USER = ""
+HTTPAUTH_PASS = ""
+HTTPAUTH_DOMAIN = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_DBM_MODULE = "dbm"

tests/test_downloadermiddleware_httpauth.py

@@ -2,8 +2,25 @@ import pytest
 from w3lib.http import basic_auth_header
 
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+
+_DOMAIN_NOT_SET = object()
+
+
+def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
+    settings: dict = {
+        "HTTPAUTH_USER": user,
+        "HTTPAUTH_PASS": passwd,
+    }
+    if domain is not _DOMAIN_NOT_SET:
+        settings["HTTPAUTH_DOMAIN"] = domain
+    return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings))
+
+
+# --- Spider attribute tests (deprecated) ---
 
 
 class LegacySpider(Spider):
@@ -23,61 +40,135 @@ class AnyDomainSpider(Spider):
     http_auth_domain = None
 
 
-class TestHttpAuthMiddlewareLegacy:
+class TestHttpAuthMiddlewareLegacySpiderAttr:
-    def setup_method(self):
+    def test_missing_domain_raises(self):
-        self.spider = LegacySpider("foo")
-
-    def test_auth(self):
         mw = HttpAuthMiddleware()
-        with pytest.raises(AttributeError):
+        with pytest.warns(ScrapyDeprecationWarning), pytest.raises(AttributeError):
-            mw.spider_opened(self.spider)
+            mw.spider_opened(LegacySpider("foo"))
 
+    def test_domain_spider(self):
+        mw = HttpAuthMiddleware()
+        with pytest.warns(ScrapyDeprecationWarning):
+            mw.spider_opened(DomainSpider("foo"))
+        req = Request("http://example.com/")
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
 
-class TestHttpAuthMiddleware:
+    def test_no_auth_wrong_domain(self):
-    def setup_method(self):
+        mw = HttpAuthMiddleware()
-        self.mw = HttpAuthMiddleware()
+        with pytest.warns(ScrapyDeprecationWarning):
-        spider = DomainSpider("foo")
+            mw.spider_opened(DomainSpider("foo"))
-        self.mw.spider_opened(spider)
+        req = Request("http://other.com/")
-
+        mw.process_request(req)
-    def teardown_method(self):
-        del self.mw
-
-    def test_no_auth(self):
-        req = Request("http://example-noauth.com/")
-        assert self.mw.process_request(req) is None
         assert "Authorization" not in req.headers
 
-    def test_auth_domain(self):
+    def test_any_domain_spider(self):
+        mw = HttpAuthMiddleware()
+        with pytest.warns(ScrapyDeprecationWarning):
+            mw.spider_opened(AnyDomainSpider("foo"))
+        req = Request("http://anywhere.com/")
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
+
+
+# --- Settings-based tests ---
+
+
+class TestHttpAuthMiddlewareSettings:
+    def test_no_auth(self):
+        mw = make_mw()
         req = Request("http://example.com/")
-        assert self.mw.process_request(req) is None
+        mw.process_request(req)
+        assert "Authorization" not in req.headers
+
+    def test_auth_without_domain_raises(self):
+        with pytest.raises(ValueError, match="HTTPAUTH_DOMAIN"):
+            make_mw(user="foo", passwd="bar")
+
+    def test_auth_all_domains(self):
+        mw = make_mw(user="foo", passwd="bar", domain=None)
+        req = Request("http://example.com/")
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
+
+    def test_auth_domain_match(self):
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
+        req = Request("http://example.com/")
+        mw.process_request(req)
         assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
 
     def test_auth_subdomain(self):
-        req = Request("http://foo.example.com/")
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
-        assert self.mw.process_request(req) is None
+        req = Request("http://sub.example.com/")
+        mw.process_request(req)
         assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
 
+    def test_no_auth_wrong_domain(self):
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
+        req = Request("http://other.com/")
+        mw.process_request(req)
+        assert "Authorization" not in req.headers
+
     def test_auth_already_set(self):
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
         req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
-        assert self.mw.process_request(req) is None
+        mw.process_request(req)
         assert req.headers["Authorization"] == b"Digest 123"
 
 
-class TestHttpAuthAnyMiddleware:
+# --- Per-request meta tests ---
-    def setup_method(self):
-        self.mw = HttpAuthMiddleware()
-        spider = AnyDomainSpider("foo")
-        self.mw.spider_opened(spider)
 
-    def teardown_method(self):
-        del self.mw
 
-    def test_auth(self):
+class TestHttpAuthMiddlewareMeta:
-        req = Request("http://example.com/")
+    def test_meta_auth_no_domain(self):
-        assert self.mw.process_request(req) is None
+        mw = make_mw()
-        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
+        req = Request("http://example.com/", meta={"http_user": "u", "http_pass": "p"})
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("u", "p")
 
-    def test_auth_already_set(self):
+    def test_meta_auth_domain_match(self):
-        req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
+        mw = make_mw()
-        assert self.mw.process_request(req) is None
+        req = Request(
+            "http://example.com/",
+            meta={
+                "http_user": "u",
+                "http_pass": "p",
+                "http_auth_domain": "example.com",
+            },
+        )
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("u", "p")
+
+    def test_meta_auth_domain_no_match(self):
+        mw = make_mw()
+        req = Request(
+            "http://other.com/",
+            meta={
+                "http_user": "u",
+                "http_pass": "p",
+                "http_auth_domain": "example.com",
+            },
+        )
+        mw.process_request(req)
+        assert "Authorization" not in req.headers
+
+    def test_meta_overrides_middleware(self):
+        mw = make_mw(user="mw_user", passwd="mw_pass", domain="example.com")
+        req = Request(
+            "http://example.com/",
+            meta={"http_user": "meta_user", "http_pass": "meta_pass"},
+        )
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header(
+            "meta_user", "meta_pass"
+        )
+
+    def test_meta_already_set(self):
+        mw = make_mw()
+        req = Request(
+            "http://example.com/",
+            headers={"Authorization": "Digest 123"},
+            meta={"http_user": "u", "http_pass": "p"},
+        )
+        mw.process_request(req)
         assert req.headers["Authorization"] == b"Digest 123"