diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8cb29deff..0c1af5276 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -307,26 +307,15 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware authenticates requests using `Basic access authentication`_ + (aka. HTTP auth). - To enable HTTP authentication for a spider, set the ``http_user`` and - ``http_pass`` spider attributes to the authentication data and the - ``http_auth_domain`` spider attribute to the domain which requires this - authentication (its subdomains will be also handled in the same way). - You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but you risk leaking your authentication - credentials to unrelated domains. + Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and + :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override + the credentials per request via :attr:`~scrapy.Request.meta` keys + :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`. - .. warning:: - In previous Scrapy versions HttpAuthMiddleware sent the authentication - data with all requests, which is a security problem if the spider - makes requests to several different domains. Currently if the - ``http_auth_domain`` attribute is not set, the middleware will use the - domain of the first request, which will work for some spiders but not - for others. In the future the middleware will produce an error instead. - - Example: + Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`): .. code-block:: python @@ -334,13 +323,62 @@ HttpAuthMiddleware class SomeIntranetSiteSpider(CrawlSpider): - http_user = "someuser" - http_pass = "somepass" - http_auth_domain = "intranet.example.com" name = "intranet.example.com" + custom_settings = { + "HTTPAUTH_USER": "someuser", + "HTTPAUTH_PASS": "somepass", + "HTTPAUTH_DOMAIN": "intranet.example.com", + } # .. rest of the spider code omitted ... + Example using per-request meta: + + .. code-block:: python + + async def start(self): + yield Request( + "https://intranet.example.com/protected/", + meta={ + "http_user": "someuser", + "http_pass": "somepass", + "http_auth_domain": "intranet.example.com", + }, + ) + +.. setting:: HTTPAUTH_USER + +HTTPAUTH_USER +~~~~~~~~~~~~~ + +Default: ``""`` + +The username to use for HTTP basic authentication, applied to all requests +whose URL matches :setting:`HTTPAUTH_DOMAIN`. + +.. setting:: HTTPAUTH_PASS + +HTTPAUTH_PASS +~~~~~~~~~~~~~ + +Default: ``""`` + +The password to use for HTTP basic authentication. + +.. setting:: HTTPAUTH_DOMAIN + +HTTPAUTH_DOMAIN +~~~~~~~~~~~~~~~ + +Default: ``None`` + +The domain (and its subdomains) to which HTTP basic authentication credentials +are sent. Set to ``None`` to send credentials with all requests, but be aware +that this risks leaking credentials to unrelated domains. + +This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` +or :setting:`HTTPAUTH_PASS` is set. + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 67b435067..700238fe9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -724,6 +724,9 @@ Those are: * :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` +* :reqmeta:`http_auth_domain` +* :reqmeta:`http_pass` +* :reqmeta:`http_user` * :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` @@ -806,6 +809,27 @@ give_up_log_level :ref:`Logging level ` used for the message logged when a request exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. +.. reqmeta:: http_auth_domain + +http_auth_domain +---------------- + +Overrides :setting:`HTTPAUTH_DOMAIN` for this request. + +.. reqmeta:: http_pass + +http_pass +--------- + +Overrides :setting:`HTTPAUTH_PASS` for this request. + +.. reqmeta:: http_user + +http_user +--------- + +Overrides :setting:`HTTPAUTH_USER` for this request. + .. reqmeta:: max_retry_times max_retry_times diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index bcef9d5f6..506daf930 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -354,11 +354,6 @@ Otherwise, you would cause iteration over a ``start_urls`` string (a very common python pitfall) resulting in each character being seen as a separate url. -A valid use case is to set the http auth credentials -used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`:: - - scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword - Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index c28c93d4e..b09abdadd 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -6,11 +6,14 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations +import warnings from typing import TYPE_CHECKING from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.url import url_is_from_any_domain @@ -23,12 +26,28 @@ if TYPE_CHECKING: class HttpAuthMiddleware: - """Set Basic HTTP Authorization header - (http_user and http_pass spider class attributes)""" + """Set Basic HTTP Authorization header.""" + + def __init__(self) -> None: + self._auth: bytes | None = None + self._domain: str | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: o = cls() + usr = crawler.settings.get("HTTPAUTH_USER", "") + pwd = crawler.settings.get("HTTPAUTH_PASS", "") + if usr or pwd: + domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0 + if domain_priority <= SETTINGS_PRIORITIES["default"]: + raise ValueError( + "HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS " + "is configured. Set it to a domain (e.g. 'example.com') to restrict " + "credentials to that domain, or set it to None to send credentials " + "with all requests." + ) + o._auth = basic_auth_header(usr, pwd) + o._domain = crawler.settings.get("HTTPAUTH_DOMAIN") crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o @@ -36,18 +55,34 @@ class HttpAuthMiddleware: usr = getattr(spider, "http_user", "") pwd = getattr(spider, "http_pass", "") if usr or pwd: - self.auth = basic_auth_header(usr, pwd) - self.domain = spider.http_auth_domain # type: ignore[attr-defined] + warnings.warn( + "Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings " + "instead of the http_user, http_pass, and http_auth_domain spider " + "attributes. Support for the spider attributes will be removed in a " + "future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._auth = basic_auth_header(usr, pwd) + self._domain = spider.http_auth_domain # type: ignore[attr-defined] @_warn_spider_arg def process_request( self, request: Request, spider: Spider | None = None ) -> Request | Response | None: - auth = getattr(self, "auth", None) - if ( - auth - and b"Authorization" not in request.headers - and (not self.domain or url_is_from_any_domain(request.url, [self.domain])) + if b"Authorization" in request.headers: + return None + # Per-request meta overrides + usr = request.meta.get("http_user", "") + pwd = request.meta.get("http_pass", "") + if usr or pwd: + domain = request.meta.get("http_auth_domain") + if not domain or url_is_from_any_domain(request.url, [domain]): + request.headers[b"Authorization"] = basic_auth_header(usr, pwd) + return None + # Middleware-level auth + if self._auth and ( + not self._domain or url_is_from_any_domain(request.url, [self._domain]) ): - request.headers[b"Authorization"] = auth + request.headers[b"Authorization"] = self._auth return None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e8a9400d7..049921ba7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -107,6 +107,9 @@ __all__ = [ "FTP_PASSWORD", "FTP_USER", "GCS_PROJECT_ID", + "HTTPAUTH_DOMAIN", + "HTTPAUTH_PASS", + "HTTPAUTH_USER", "HTTPCACHE_ALWAYS_STORE", "HTTPCACHE_DBM_MODULE", "HTTPCACHE_DIR", @@ -397,6 +400,10 @@ FTP_PASSWORD = "guest" # noqa: S105 GCS_PROJECT_ID = None +HTTPAUTH_USER = "" +HTTPAUTH_PASS = "" +HTTPAUTH_DOMAIN = None + HTTPCACHE_ENABLED = False HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_DBM_MODULE = "dbm" diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 522a3002f..827133d2e 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -2,8 +2,25 @@ import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + +_DOMAIN_NOT_SET = object() + + +def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET): + settings: dict = { + "HTTPAUTH_USER": user, + "HTTPAUTH_PASS": passwd, + } + if domain is not _DOMAIN_NOT_SET: + settings["HTTPAUTH_DOMAIN"] = domain + return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings)) + + +# --- Spider attribute tests (deprecated) --- class LegacySpider(Spider): @@ -23,61 +40,135 @@ class AnyDomainSpider(Spider): http_auth_domain = None -class TestHttpAuthMiddlewareLegacy: - def setup_method(self): - self.spider = LegacySpider("foo") - - def test_auth(self): +class TestHttpAuthMiddlewareLegacySpiderAttr: + def test_missing_domain_raises(self): mw = HttpAuthMiddleware() - with pytest.raises(AttributeError): - mw.spider_opened(self.spider) + with pytest.warns(ScrapyDeprecationWarning), pytest.raises(AttributeError): + mw.spider_opened(LegacySpider("foo")) + def test_domain_spider(self): + mw = HttpAuthMiddleware() + with pytest.warns(ScrapyDeprecationWarning): + mw.spider_opened(DomainSpider("foo")) + req = Request("http://example.com/") + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") -class TestHttpAuthMiddleware: - def setup_method(self): - self.mw = HttpAuthMiddleware() - spider = DomainSpider("foo") - self.mw.spider_opened(spider) - - def teardown_method(self): - del self.mw - - def test_no_auth(self): - req = Request("http://example-noauth.com/") - assert self.mw.process_request(req) is None + def test_no_auth_wrong_domain(self): + mw = HttpAuthMiddleware() + with pytest.warns(ScrapyDeprecationWarning): + mw.spider_opened(DomainSpider("foo")) + req = Request("http://other.com/") + mw.process_request(req) assert "Authorization" not in req.headers - def test_auth_domain(self): + def test_any_domain_spider(self): + mw = HttpAuthMiddleware() + with pytest.warns(ScrapyDeprecationWarning): + mw.spider_opened(AnyDomainSpider("foo")) + req = Request("http://anywhere.com/") + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") + + +# --- Settings-based tests --- + + +class TestHttpAuthMiddlewareSettings: + def test_no_auth(self): + mw = make_mw() req = Request("http://example.com/") - assert self.mw.process_request(req) is None + mw.process_request(req) + assert "Authorization" not in req.headers + + def test_auth_without_domain_raises(self): + with pytest.raises(ValueError, match="HTTPAUTH_DOMAIN"): + make_mw(user="foo", passwd="bar") + + def test_auth_all_domains(self): + mw = make_mw(user="foo", passwd="bar", domain=None) + req = Request("http://example.com/") + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") + + def test_auth_domain_match(self): + mw = make_mw(user="foo", passwd="bar", domain="example.com") + req = Request("http://example.com/") + mw.process_request(req) assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_subdomain(self): - req = Request("http://foo.example.com/") - assert self.mw.process_request(req) is None + mw = make_mw(user="foo", passwd="bar", domain="example.com") + req = Request("http://sub.example.com/") + mw.process_request(req) assert req.headers["Authorization"] == basic_auth_header("foo", "bar") + def test_no_auth_wrong_domain(self): + mw = make_mw(user="foo", passwd="bar", domain="example.com") + req = Request("http://other.com/") + mw.process_request(req) + assert "Authorization" not in req.headers + def test_auth_already_set(self): + mw = make_mw(user="foo", passwd="bar", domain="example.com") req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req) is None + mw.process_request(req) assert req.headers["Authorization"] == b"Digest 123" -class TestHttpAuthAnyMiddleware: - def setup_method(self): - self.mw = HttpAuthMiddleware() - spider = AnyDomainSpider("foo") - self.mw.spider_opened(spider) +# --- Per-request meta tests --- - def teardown_method(self): - del self.mw - def test_auth(self): - req = Request("http://example.com/") - assert self.mw.process_request(req) is None - assert req.headers["Authorization"] == basic_auth_header("foo", "bar") +class TestHttpAuthMiddlewareMeta: + def test_meta_auth_no_domain(self): + mw = make_mw() + req = Request("http://example.com/", meta={"http_user": "u", "http_pass": "p"}) + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("u", "p") - def test_auth_already_set(self): - req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req) is None + def test_meta_auth_domain_match(self): + mw = make_mw() + req = Request( + "http://example.com/", + meta={ + "http_user": "u", + "http_pass": "p", + "http_auth_domain": "example.com", + }, + ) + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("u", "p") + + def test_meta_auth_domain_no_match(self): + mw = make_mw() + req = Request( + "http://other.com/", + meta={ + "http_user": "u", + "http_pass": "p", + "http_auth_domain": "example.com", + }, + ) + mw.process_request(req) + assert "Authorization" not in req.headers + + def test_meta_overrides_middleware(self): + mw = make_mw(user="mw_user", passwd="mw_pass", domain="example.com") + req = Request( + "http://example.com/", + meta={"http_user": "meta_user", "http_pass": "meta_pass"}, + ) + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header( + "meta_user", "meta_pass" + ) + + def test_meta_already_set(self): + mw = make_mw() + req = Request( + "http://example.com/", + headers={"Authorization": "Digest 123"}, + meta={"http_user": "u", "http_pass": "p"}, + ) + mw.process_request(req) assert req.headers["Authorization"] == b"Digest 123"