Merge branch 'master' into py3.15

docs/topics/asyncio.rst

@@ -4,19 +4,27 @@
 asyncio
 =======
 
-Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
-asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
-:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
+Scrapy supports :mod:`asyncio` natively. New projects created with
+:command:`scrapy startproject` have asyncio enabled by default, and you can use
+:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
+<coroutines>`.
+
+The rest of this page covers advanced topics. If you are starting a new project,
+no additional setup is needed.
 
 
 .. _install-asyncio:
 
-Installing the asyncio reactor
-==============================
+Configuring the asyncio reactor
+===============================
 
-To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs
-to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,
-which is the default value.
+New projects generated with :command:`scrapy startproject` have the asyncio
+reactor configured by default. No manual setup is needed.
+
+The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
+uses. Its default value is
+``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables
+:mod:`asyncio` support.
 
 If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or
 :class:`~scrapy.crawler.CrawlerRunner`, you also need to

docs/topics/coroutines.rst

@@ -267,6 +267,6 @@ You can also send multiple requests in parallel:
             responses = await asyncio.gather(*tasks)
             yield {
                 "h1": response.css("h1::text").get(),
-                "price": responses[0][1].css(".price::text").get(),
-                "price2": responses[1][1].css(".color::text").get(),
+                "price": responses[0].css(".price::text").get(),
+                "price2": responses[1].css(".color::text").get(),
             }

docs/topics/downloader-middleware.rst

@@ -307,26 +307,15 @@ HttpAuthMiddleware
 
 .. class:: HttpAuthMiddleware
 
-    This middleware authenticates all requests generated from certain spiders
-    using `Basic access authentication`_ (aka. HTTP auth).
+    This middleware authenticates requests using `Basic access authentication`_
+    (aka. HTTP auth).
 
-    To enable HTTP authentication for a spider, set the ``http_user`` and
-    ``http_pass`` spider attributes to the authentication data and the
-    ``http_auth_domain`` spider attribute to the domain which requires this
-    authentication (its subdomains will be also handled in the same way).
-    You can set ``http_auth_domain`` to ``None`` to enable the
-    authentication for all requests but you risk leaking your authentication
-    credentials to unrelated domains.
+    Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and
+    :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override
+    the credentials per request via :attr:`~scrapy.Request.meta` keys
+    :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`.
 
-    .. warning::
-        In previous Scrapy versions HttpAuthMiddleware sent the authentication
-        data with all requests, which is a security problem if the spider
-        makes requests to several different domains. Currently if the
-        ``http_auth_domain`` attribute is not set, the middleware will use the
-        domain of the first request, which will work for some spiders but not
-        for others. In the future the middleware will produce an error instead.
-
-    Example:
+    Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`):
 
     .. code-block:: python
 
@@ -334,13 +323,62 @@ HttpAuthMiddleware
 
 
         class SomeIntranetSiteSpider(CrawlSpider):
-            http_user = "someuser"
-            http_pass = "somepass"
-            http_auth_domain = "intranet.example.com"
             name = "intranet.example.com"
+            custom_settings = {
+                "HTTPAUTH_USER": "someuser",
+                "HTTPAUTH_PASS": "somepass",
+                "HTTPAUTH_DOMAIN": "intranet.example.com",
+            }
 
             # .. rest of the spider code omitted ...
 
+    Example using per-request meta:
+
+    .. code-block:: python
+
+        async def start(self):
+            yield Request(
+                "https://intranet.example.com/protected/",
+                meta={
+                    "http_user": "someuser",
+                    "http_pass": "somepass",
+                    "http_auth_domain": "intranet.example.com",
+                },
+            )
+
+.. setting:: HTTPAUTH_USER
+
+HTTPAUTH_USER
+~~~~~~~~~~~~~
+
+Default: ``""``
+
+The username to use for HTTP basic authentication, applied to all requests
+whose URL matches :setting:`HTTPAUTH_DOMAIN`.
+
+.. setting:: HTTPAUTH_PASS
+
+HTTPAUTH_PASS
+~~~~~~~~~~~~~
+
+Default: ``""``
+
+The password to use for HTTP basic authentication.
+
+.. setting:: HTTPAUTH_DOMAIN
+
+HTTPAUTH_DOMAIN
+~~~~~~~~~~~~~~~
+
+Default: ``None``
+
+The domain (and its subdomains) to which HTTP basic authentication credentials
+are sent. Set to ``None`` to send credentials with all requests, but be aware
+that this risks leaking credentials to unrelated domains.
+
+This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
+or :setting:`HTTPAUTH_PASS` is set.
+
 .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
 
 

docs/topics/extensions.rst

@@ -149,8 +149,6 @@ The following stats are collected:
   (e.g. ``item_dropped_reasons_count/DropItem``).
 * ``response_received_count``: total number of HTTP responses received.
 
-.. _topics-extensions-ref-telnetconsole:
-
 Log Count extension
 ~~~~~~~~~~~~~~~~~~~
 
@@ -159,6 +157,8 @@ Log Count extension
 
 .. autoclass:: LogCount
 
+.. _topics-extensions-ref-telnetconsole:
+
 Telnet console extension
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/topics/jobs.rst

@@ -104,6 +104,15 @@ If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
 
+.. note:: Because requests are serialized with :mod:`pickle`, the objects you
+    store on a request, such as the values of its
+    :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta`
+    dictionaries, are deep-copied when the request is written to and later read
+    back from the job directory. As a result, the callback receives a *copy* of
+    those objects rather than the original ones, and changes made to the copy are
+    not reflected in the original object. Keep this in mind if you rely on
+    sharing mutable state through ``cb_kwargs`` or ``meta``.
+
 .. _job-dir-contents:
 
 Job directory contents

docs/topics/practices.rst

@@ -387,6 +387,26 @@ crawl::
     curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2
     curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3
 
+.. _large-project-startup:
+
+Reducing startup time in large projects
+=======================================
+
+When running a spider with ``scrapy crawl``, Scrapy loads all modules listed in
+:setting:`SPIDER_MODULES` to find the target spider. In large projects with
+many spiders, this can noticeably increase startup time and memory usage.
+
+To avoid loading every spider module, override :setting:`SPIDER_MODULES` on the
+command line to point only to the module that contains the spider you want to
+run:
+
+.. code-block:: shell
+
+    scrapy crawl myspider -s SPIDER_MODULES=myproject.spiders.myspider
+
+Because :setting:`SPIDER_MODULES` is a list setting, you can include multiple
+modules by separating them with commas.
+
 .. _bans:
 
 Avoiding getting banned

docs/topics/request-response.rst

@@ -188,6 +188,13 @@ Request objects
         ``failure.request.cb_kwargs`` in the request's errback. For more information,
         see :ref:`errback-cb_kwargs`.
 
+        .. note:: When :setting:`JOBDIR` is set, requests are serialized to disk
+            with :mod:`pickle` (see :ref:`request-serialization`). As a result,
+            the callback receives a deep copy of any object stored in
+            ``cb_kwargs``, so mutating such an object in the callback does not
+            affect the original. Avoid relying on shared mutable state passed
+            through ``cb_kwargs`` in that case.
+
     .. attribute:: Request.meta
        :value: {}
 
@@ -717,6 +724,9 @@ Those are:
 * :reqmeta:`give_up_log_level`
 * :reqmeta:`handle_httpstatus_all`
 * :reqmeta:`handle_httpstatus_list`
+* :reqmeta:`http_auth_domain`
+* :reqmeta:`http_pass`
+* :reqmeta:`http_user`
 * :reqmeta:`is_start_request`
 * :reqmeta:`max_retry_times`
 * :reqmeta:`proxy`
@@ -799,6 +809,27 @@ give_up_log_level
 :ref:`Logging level <levels>` used for the message logged when a request
 exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
 
+.. reqmeta:: http_auth_domain
+
+http_auth_domain
+----------------
+
+Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
+
+.. reqmeta:: http_pass
+
+http_pass
+---------
+
+Overrides :setting:`HTTPAUTH_PASS` for this request.
+
+.. reqmeta:: http_user
+
+http_user
+---------
+
+Overrides :setting:`HTTPAUTH_USER` for this request.
+
 .. reqmeta:: max_retry_times
 
 max_retry_times
@@ -979,7 +1010,7 @@ Response objects
 
         A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains
         the response headers. Values can be accessed using
-        :meth:`~scrapy.http.headers.Headers.get` to return the first header value with
+        :meth:`~scrapy.http.headers.Headers.get` to return the last header value with
         the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
         all header values with the specified name. For example, this call will give you
         all cookies in the headers::

docs/topics/settings.rst

@@ -734,8 +734,7 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it.
 
     Handling of this setting needs to be implemented inside the :ref:`download
     handler <topics-download-handlers>`, so it's not guaranteed to be supported
-    by all 3rd-party handlers. It's currently unsupported by
-    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
+    by all 3rd-party handlers.
 
 .. setting:: DOWNLOAD_TLS_MAX_VERSION
 

docs/topics/spiders.rst

@@ -354,11 +354,6 @@ Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
 resulting in each character being seen as a separate url.
 
-A valid use case is to set the http auth credentials
-used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`::
-
-    scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword
-
 Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
 See `Scrapyd documentation`_.
 

docs/topics/telnetconsole.rst

@@ -94,8 +94,6 @@ convenience:
 +----------------+-------------------------------------------------------------------+
 | ``p``          | a shortcut to the :func:`pprint.pprint` function                  |
 +----------------+-------------------------------------------------------------------+
-| ``hpy``        | for memory debugging (see :ref:`topics-leaks`)                    |
-+----------------+-------------------------------------------------------------------+
 
 Telnet console usage examples
 =============================

scrapy/downloadermiddlewares/httpauth.py

@@ -6,11 +6,14 @@ See documentation in docs/topics/downloader-middleware.rst
 
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING
 
 from w3lib.http import basic_auth_header
 
 from scrapy import Request, Spider, signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings import SETTINGS_PRIORITIES
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.url import url_is_from_any_domain
 
@@ -23,12 +26,28 @@ if TYPE_CHECKING:
 
 
 class HttpAuthMiddleware:
-    """Set Basic HTTP Authorization header
-    (http_user and http_pass spider class attributes)"""
+    """Set Basic HTTP Authorization header."""
+
+    def __init__(self) -> None:
+        self._auth: bytes | None = None
+        self._domain: str | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         o = cls()
+        usr = crawler.settings.get("HTTPAUTH_USER", "")
+        pwd = crawler.settings.get("HTTPAUTH_PASS", "")
+        if usr or pwd:
+            domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0
+            if domain_priority <= SETTINGS_PRIORITIES["default"]:
+                raise ValueError(
+                    "HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS "
+                    "is configured. Set it to a domain (e.g. 'example.com') to restrict "
+                    "credentials to that domain, or set it to None to send credentials "
+                    "with all requests."
+                )
+            o._auth = basic_auth_header(usr, pwd)
+            o._domain = crawler.settings.get("HTTPAUTH_DOMAIN")
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         return o
 
@@ -36,18 +55,34 @@ class HttpAuthMiddleware:
         usr = getattr(spider, "http_user", "")
         pwd = getattr(spider, "http_pass", "")
         if usr or pwd:
-            self.auth = basic_auth_header(usr, pwd)
-            self.domain = spider.http_auth_domain  # type: ignore[attr-defined]
+            warnings.warn(
+                "Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings "
+                "instead of the http_user, http_pass, and http_auth_domain spider "
+                "attributes. Support for the spider attributes will be removed in a "
+                "future version of Scrapy.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self._auth = basic_auth_header(usr, pwd)
+            self._domain = spider.http_auth_domain  # type: ignore[attr-defined]
 
     @_warn_spider_arg
     def process_request(
         self, request: Request, spider: Spider | None = None
     ) -> Request | Response | None:
-        auth = getattr(self, "auth", None)
-        if (
-            auth
-            and b"Authorization" not in request.headers
-            and (not self.domain or url_is_from_any_domain(request.url, [self.domain]))
+        if b"Authorization" in request.headers:
+            return None
+        # Per-request meta overrides
+        usr = request.meta.get("http_user", "")
+        pwd = request.meta.get("http_pass", "")
+        if usr or pwd:
+            domain = request.meta.get("http_auth_domain")
+            if not domain or url_is_from_any_domain(request.url, [domain]):
+                request.headers[b"Authorization"] = basic_auth_header(usr, pwd)
+            return None
+        # Middleware-level auth
+        if self._auth and (
+            not self._domain or url_is_from_any_domain(request.url, [self._domain])
         ):
-            request.headers[b"Authorization"] = auth
+            request.headers[b"Authorization"] = self._auth
         return None

scrapy/link.py

@@ -39,7 +39,7 @@ class Link:
 
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, Link):
-            raise NotImplementedError
+            return NotImplemented
         return (
             self.url == other.url
             and self.text == other.text

scrapy/loader/__init__.py

@@ -27,7 +27,7 @@ class ItemLoader(itemloaders.ItemLoader):
     :param item: The item instance to populate using subsequent calls to
         :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
         or :meth:`~ItemLoader.add_value`.
-    :type item: scrapy.item.Item
+    :type item: :ref:`item object <item-types>`
 
     :param selector: The selector to extract data from, when using the
         :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or

scrapy/settings/default_settings.py

@@ -107,6 +107,9 @@ __all__ = [
     "FTP_PASSWORD",
     "FTP_USER",
     "GCS_PROJECT_ID",
+    "HTTPAUTH_DOMAIN",
+    "HTTPAUTH_PASS",
+    "HTTPAUTH_USER",
     "HTTPCACHE_ALWAYS_STORE",
     "HTTPCACHE_DBM_MODULE",
     "HTTPCACHE_DIR",
@@ -397,6 +400,10 @@ FTP_PASSWORD = "guest"  # noqa: S105
 
 GCS_PROJECT_ID = None
 
+HTTPAUTH_USER = ""
+HTTPAUTH_PASS = ""
+HTTPAUTH_DOMAIN = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_DBM_MODULE = "dbm"

scrapy/shell.py

@@ -30,7 +30,6 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.misc import load_object
-from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 from scrapy.utils.response import open_in_browser
 
 if TYPE_CHECKING:
@@ -76,8 +75,8 @@ if TYPE_CHECKING:
 # running event loop.
 #
 # Side note: it should be possible to remove _request_deferred() by using
-# engine.download() instead of engine.schedule(), losing the usual stuff like
-# spider middlewares (none of which should be important).
+# engine.download_async() instead of engine.schedule(), losing the usual stuff
+# like spider middlewares (none of which should be important).
 #
 # Other architecture problems:
 # * scrapy.cmdline.execute() creates an AsyncCrawlerProcess instance which
@@ -191,10 +190,6 @@ class Shell:
 
         Runs in the reactor thread.
         """
-        if self._use_reactor and is_asyncio_reactor_installed():
-            # set the asyncio event loop for the current thread
-            event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
-            set_asyncio_event_loop(event_loop_path)
         if not self.spider:
             await self._open_spider(spider)
         assert self.crawler.engine is not None

scrapy/utils/reactor.py

@@ -33,6 +33,8 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
 
     if len(portrange) > 2:
         raise ValueError(f"invalid portrange: {portrange}")
+    if len(portrange) == 2 and portrange[0] > portrange[1]:
+        raise ValueError(f"invalid portrange: {portrange}")
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)  # type: ignore[no-any-return]
     if len(portrange) == 1:
@@ -185,7 +187,7 @@ def verify_installed_reactor(reactor_path: str) -> None:
 
 
 def verify_installed_asyncio_event_loop(loop_path: str) -> None:
-    """Raise :exc:`RuntimeError` if the even loop of the installed
+    """Raise :exc:`RuntimeError` if the event loop of the installed
     :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
     does not match the specified import path or if no reactor is installed."""
     if not is_reactor_installed():

tests/test_addons.py

@@ -71,6 +71,37 @@ class TestAddonManager:
         manager = crawler.addons
         assert not manager.addons
 
+    def test_notconfigured_with_args(self):
+        class NotConfiguredAddon:
+            def update_settings(self, settings):
+                raise NotConfigured("addon disabled reason")
+
+        settings_dict = {
+            "ADDONS": {NotConfiguredAddon: 0},
+        }
+        with patch("scrapy.addons.logger") as logger_mock:
+            crawler = get_crawler(settings_dict=settings_dict)
+        assert not crawler.addons.addons
+        logger_mock.warning.assert_called_once_with(
+            "Disabled %(clspath)s: %(eargs)s",
+            {"clspath": NotConfiguredAddon, "eargs": "addon disabled reason"},
+            extra={"crawler": crawler},
+        )
+
+    def test_no_update_settings(self):
+        class PreCrawlerOnlyAddon:
+            @classmethod
+            def update_pre_crawler_settings(cls, settings):
+                settings.set("PRE_CRAWLER_KEY", "value", priority="addon")
+
+        settings_dict = {
+            "ADDONS": {PreCrawlerOnlyAddon: 0},
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        manager = crawler.addons
+        assert len(manager.addons) == 1
+        assert isinstance(manager.addons[0], PreCrawlerOnlyAddon)
+
     def test_load_settings_order(self):
         # Get three addons with different settings
         addonlist = []

tests/test_downloadermiddleware_httpauth.py

@@ -2,8 +2,25 @@ import pytest
 from w3lib.http import basic_auth_header
 
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+
+_DOMAIN_NOT_SET = object()
+
+
+def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
+    settings: dict = {
+        "HTTPAUTH_USER": user,
+        "HTTPAUTH_PASS": passwd,
+    }
+    if domain is not _DOMAIN_NOT_SET:
+        settings["HTTPAUTH_DOMAIN"] = domain
+    return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings))
+
+
+# --- Spider attribute tests (deprecated) ---
 
 
 class LegacySpider(Spider):
@@ -23,61 +40,135 @@ class AnyDomainSpider(Spider):
     http_auth_domain = None
 
 
-class TestHttpAuthMiddlewareLegacy:
-    def setup_method(self):
-        self.spider = LegacySpider("foo")
-
-    def test_auth(self):
+class TestHttpAuthMiddlewareLegacySpiderAttr:
+    def test_missing_domain_raises(self):
         mw = HttpAuthMiddleware()
-        with pytest.raises(AttributeError):
-            mw.spider_opened(self.spider)
+        with pytest.warns(ScrapyDeprecationWarning), pytest.raises(AttributeError):
+            mw.spider_opened(LegacySpider("foo"))
 
+    def test_domain_spider(self):
+        mw = HttpAuthMiddleware()
+        with pytest.warns(ScrapyDeprecationWarning):
+            mw.spider_opened(DomainSpider("foo"))
+        req = Request("http://example.com/")
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
 
-class TestHttpAuthMiddleware:
-    def setup_method(self):
-        self.mw = HttpAuthMiddleware()
-        spider = DomainSpider("foo")
-        self.mw.spider_opened(spider)
-
-    def teardown_method(self):
-        del self.mw
-
-    def test_no_auth(self):
-        req = Request("http://example-noauth.com/")
-        assert self.mw.process_request(req) is None
+    def test_no_auth_wrong_domain(self):
+        mw = HttpAuthMiddleware()
+        with pytest.warns(ScrapyDeprecationWarning):
+            mw.spider_opened(DomainSpider("foo"))
+        req = Request("http://other.com/")
+        mw.process_request(req)
         assert "Authorization" not in req.headers
 
-    def test_auth_domain(self):
+    def test_any_domain_spider(self):
+        mw = HttpAuthMiddleware()
+        with pytest.warns(ScrapyDeprecationWarning):
+            mw.spider_opened(AnyDomainSpider("foo"))
+        req = Request("http://anywhere.com/")
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
+
+
+# --- Settings-based tests ---
+
+
+class TestHttpAuthMiddlewareSettings:
+    def test_no_auth(self):
+        mw = make_mw()
         req = Request("http://example.com/")
-        assert self.mw.process_request(req) is None
+        mw.process_request(req)
+        assert "Authorization" not in req.headers
+
+    def test_auth_without_domain_raises(self):
+        with pytest.raises(ValueError, match="HTTPAUTH_DOMAIN"):
+            make_mw(user="foo", passwd="bar")
+
+    def test_auth_all_domains(self):
+        mw = make_mw(user="foo", passwd="bar", domain=None)
+        req = Request("http://example.com/")
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
+
+    def test_auth_domain_match(self):
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
+        req = Request("http://example.com/")
+        mw.process_request(req)
         assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
 
     def test_auth_subdomain(self):
-        req = Request("http://foo.example.com/")
-        assert self.mw.process_request(req) is None
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
+        req = Request("http://sub.example.com/")
+        mw.process_request(req)
         assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
 
+    def test_no_auth_wrong_domain(self):
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
+        req = Request("http://other.com/")
+        mw.process_request(req)
+        assert "Authorization" not in req.headers
+
     def test_auth_already_set(self):
+        mw = make_mw(user="foo", passwd="bar", domain="example.com")
         req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
-        assert self.mw.process_request(req) is None
+        mw.process_request(req)
         assert req.headers["Authorization"] == b"Digest 123"
 
 
-class TestHttpAuthAnyMiddleware:
-    def setup_method(self):
-        self.mw = HttpAuthMiddleware()
-        spider = AnyDomainSpider("foo")
-        self.mw.spider_opened(spider)
+# --- Per-request meta tests ---
 
-    def teardown_method(self):
-        del self.mw
 
-    def test_auth(self):
-        req = Request("http://example.com/")
-        assert self.mw.process_request(req) is None
-        assert req.headers["Authorization"] == basic_auth_header("foo", "bar")
+class TestHttpAuthMiddlewareMeta:
+    def test_meta_auth_no_domain(self):
+        mw = make_mw()
+        req = Request("http://example.com/", meta={"http_user": "u", "http_pass": "p"})
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("u", "p")
 
-    def test_auth_already_set(self):
-        req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
-        assert self.mw.process_request(req) is None
+    def test_meta_auth_domain_match(self):
+        mw = make_mw()
+        req = Request(
+            "http://example.com/",
+            meta={
+                "http_user": "u",
+                "http_pass": "p",
+                "http_auth_domain": "example.com",
+            },
+        )
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header("u", "p")
+
+    def test_meta_auth_domain_no_match(self):
+        mw = make_mw()
+        req = Request(
+            "http://other.com/",
+            meta={
+                "http_user": "u",
+                "http_pass": "p",
+                "http_auth_domain": "example.com",
+            },
+        )
+        mw.process_request(req)
+        assert "Authorization" not in req.headers
+
+    def test_meta_overrides_middleware(self):
+        mw = make_mw(user="mw_user", passwd="mw_pass", domain="example.com")
+        req = Request(
+            "http://example.com/",
+            meta={"http_user": "meta_user", "http_pass": "meta_pass"},
+        )
+        mw.process_request(req)
+        assert req.headers["Authorization"] == basic_auth_header(
+            "meta_user", "meta_pass"
+        )
+
+    def test_meta_already_set(self):
+        mw = make_mw()
+        req = Request(
+            "http://example.com/",
+            headers={"Authorization": "Digest 123"},
+            meta={"http_user": "u", "http_pass": "p"},
+        )
+        mw.process_request(req)
         assert req.headers["Authorization"] == b"Digest 123"

tests/test_extension_telnet.py

@@ -53,3 +53,9 @@ class TestTelnetExtension:
         d = portal.login(creds, None, ITelnetProtocol)
         yield d
         console.stop_listening()
+
+    def test_invalid_reversed_portrange(self):
+        settings = {"TELNETCONSOLE_PORT": [2, 1]}
+        console = TelnetConsole(get_crawler(settings_dict=settings))
+        with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"):
+            console.start_listening()

tests/test_item.py

@@ -48,6 +48,18 @@ class TestItem:
         with pytest.raises(KeyError):
             i["field"]
 
+    def test_delitem(self):
+        class TestItem(Item):
+            name = Field()
+
+        i = TestItem(name="John")
+        del i["name"]
+        with pytest.raises(KeyError):
+            i["name"]
+
+        with pytest.raises(KeyError):
+            del i["name"]
+
     def test_repr(self):
         class TestItem(Item):
             name = Field()

tests/test_link.py

@@ -55,3 +55,7 @@ class TestLink:
     def test_bytes_url(self):
         with pytest.raises(TypeError):
             Link(b"http://www.example.com/\xc2\xa3")
+
+    def test_eq_non_link(self):
+        url = "http://example.com"
+        assert Link(url) != url

tests/test_spiderloader/__init__.py

@@ -252,3 +252,4 @@ def test_dummy_spider_loader(spider_loader_env):
     assert not spider_loader.list()
     with pytest.raises(KeyError):
         spider_loader.load("spider1")
+    assert not spider_loader.find_by_request(Request("http://example.com"))