Bump version: 2.11.1 → 2.11.2

Co-authored-by: John Doe <johndoe@email.com>

.bandit.yml

@@ -7,14 +7,12 @@ skips:
 - B306
 - B307
 - B311
-- B320
 - B321
 - B324
 - B402  # https://github.com/scrapy/scrapy/issues/4180
 - B403
 - B404
 - B406
-- B410
 - B503
 - B603
 - B605

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.11.1
+current_version = 2.11.2
 commit = True
 tag = True
 tag_name = {new_version}

MANIFEST.in

@@ -1,9 +1,8 @@
-include README.rst
+include CODE_OF_CONDUCT.md
-include AUTHORS
+include CONTRIBUTING.md
-include INSTALL
+include INSTALL.md
-include LICENSE
-include MANIFEST.in
 include NEWS
+include SECURITY.md
 
 include scrapy/VERSION
 include scrapy/mime.types
@@ -11,16 +10,13 @@ include scrapy/mime.types
 include codecov.yml
 include conftest.py
 include pytest.ini
-include requirements-*.txt
 include tox.ini
 
 recursive-include scrapy/templates *
-recursive-include scrapy license.txt
 recursive-include docs *
 prune docs/build
 
 recursive-include extras *
-recursive-include bin *
 recursive-include tests *
 
 global-exclude __pycache__ *.py[cod]

docs/conf.py

@@ -10,7 +10,6 @@
 # serve to show the default.
 
 import sys
-from datetime import datetime
 from pathlib import Path
 
 # If your extensions are in another directory, add it here. If the directory
@@ -48,7 +47,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "Scrapy"
-copyright = f"2008–{datetime.now().year}, Scrapy developers"
+copyright = "Scrapy developers"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -227,7 +226,7 @@ latex_documents = [
 # A list of regular expressions that match URIs that should not be checked when
 # doing a linkcheck build.
 linkcheck_ignore = [
-    "http://localhost:\d+",
+    r"http://localhost:\d+",
     "http://hg.scrapy.org",
     "http://directory.google.com/",
 ]

docs/faq.rst

@@ -138,39 +138,37 @@ See previous question.
 How can I prevent memory errors due to many allowed domains?
 ------------------------------------------------------------
 
-If you have a spider with a long list of
+If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains`
-:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider
+(e.g. 50,000+), consider replacing the default
-replacing the default
+:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader
-:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
+middleware with a :ref:`custom downloader middleware
-with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
+<topics-downloader-middleware-custom>` that requires less memory. For example:
-less memory. For example:
 
 -   If your domain names are similar enough, use your own regular expression
-    instead joining the strings in
+    instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
-    :attr:`~scrapy.Spider.allowed_domains` into a complex regular
+    a complex regular expression.
-    expression.
 
 -   If you can `meet the installation requirements`_, use pyre2_ instead of
     Python’s re_ to compile your URL-filtering regular expression. See
     :issue:`1908`.
 
-See also other suggestions at `StackOverflow`_.
+See also `other suggestions at StackOverflow
+<https://stackoverflow.com/q/36440681>`__.
 
 .. note:: Remember to disable
-   :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
+   :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you
-   your custom implementation:
+   enable your custom implementation:
 
    .. code-block:: python
 
-       SPIDER_MIDDLEWARES = {
+       DOWNLOADER_MIDDLEWARES = {
-           "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
+           "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None,
-           "myproject.middlewares.CustomOffsiteMiddleware": 500,
+           "myproject.middlewares.CustomOffsiteMiddleware": 50,
        }
 
 .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
 .. _pyre2: https://github.com/andreasvc/pyre2
 .. _re: https://docs.python.org/library/re.html
-.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
 
 Can I use Basic HTTP Authentication in my spiders?
 --------------------------------------------------
@@ -206,12 +204,10 @@ I get "Filtered offsite request" messages. How can I fix them?
 Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a
 problem, so you may not need to fix them.
 
-Those messages are thrown by the Offsite Spider Middleware, which is a spider
+Those messages are thrown by
-middleware (enabled by default) whose purpose is to filter out requests to
+:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a
-domains outside the ones covered by the spider.
+downloader middleware (enabled by default) whose purpose is to filter out
-
+requests to domains outside the ones covered by the spider.
-For more info see:
-:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`.
 
 What is the recommended way to deploy a Scrapy crawler in production?
 ---------------------------------------------------------------------
@@ -409,6 +405,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
 :ref:`topics-stop-response-download` topic for additional information and examples.
 
 
+.. _faq-blank-request:
+
+How can I make a blank request?
+-------------------------------
+
+.. code-block:: python
+    
+    from scrapy import Request
+
+
+    blank_request = Request("data:,")
+
+In this case, the URL is set to a data URI scheme. Data URLs allow you to include data 
+in-line in web pages as if they were external resources. The "data:" scheme with an empty 
+content (",") essentially creates a request to a data URL without any specific content.
+
+
 Running ``runspider`` I get ``error: No spider found in file: <filename>``
 --------------------------------------------------------------------------
 

docs/news.rst

@@ -3,6 +3,117 @@
 Release notes
 =============
 
+.. _release-2.11.2:
+
+Scrapy 2.11.2 (2024-05-14)
+--------------------------
+
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+-   Redirects to non-HTTP protocols are no longer followed. Please, see the
+    `23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`)
+
+    .. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h
+
+-   The ``Authorization`` header is now dropped on redirects to a different
+    scheme (``http://`` or ``https://``) or port, even if the domain is the
+    same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more
+    information.
+
+    .. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f
+
+-   When using system proxy settings that are different for ``http://`` and
+    ``https://``, redirects to a different URL scheme will now also trigger the
+    corresponding change in proxy settings for the redirected request. Please,
+    see the `jm3v-qxmh-hxwv security advisory`_ for more information.
+    (:issue:`767`)
+
+    .. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv
+
+-   :attr:`Spider.allowed_domains <scrapy.Spider.allowed_domains>` is now
+    enforced for all requests, and not only requests from spider callbacks.
+    (:issue:`1042`, :issue:`2241`, :issue:`6358`)
+
+-   :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML
+    entities. (:issue:`6265`)
+
+-   defusedxml_ is now used to make
+    :class:`scrapy.http.request.rpc.XmlRpcRequest` more secure.
+    (:issue:`6250`, :issue:`6251`)
+
+    .. _defusedxml: https://github.com/tiran/defusedxml
+
+Bug fixes
+~~~~~~~~~
+
+-   Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in
+    favor of brotli_. (:issue:`6261`)
+
+    .. _brotli: https://github.com/google/brotli
+
+    .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli
+        instead if you can.
+
+-   Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This
+    prevents
+    :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from
+    following redirects that would not be followed by web browsers with
+    JavaScript enabled. (:issue:`6342`, :issue:`6347`)
+
+-   During :ref:`feed export <topics-feed-exports>`, do not close the
+    underlying file from :ref:`built-in post-processing plugins
+    <builtin-plugins>`.
+    (:issue:`5932`, :issue:`6178`, :issue:`6239`)
+
+-   :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    now properly applies the ``unique`` and ``canonicalize`` parameters.
+    (:issue:`3273`, :issue:`6221`)
+
+-   Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty
+    string. (:issue:`6121`, :issue:`6124`)
+
+-   Fix :attr:`Spider.logger <scrapy.Spider.logger>` not logging custom extra
+    information. (:issue:`6323`, :issue:`6324`)
+
+-   ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing
+    the UTF-8-compatible (e.g. ASCII) parts of the document.
+    (:issue:`6292`, :issue:`6298`)
+
+-   :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an
+    exception if ``default`` is ``None``.
+    (:issue:`6308`, :issue:`6310`)
+
+-   :class:`~scrapy.selector.Selector` now uses
+    :func:`scrapy.utils.response.get_base_url` to determine the base URL of a
+    given :class:`~scrapy.http.Response`. (:issue:`6265`)
+
+-   The :meth:`media_to_download` method of :ref:`media pipelines
+    <topics-media-pipeline>` now logs exceptions before stripping them.
+    (:issue:`5067`, :issue:`5068`)
+
+-   When passing a callback to the :command:`parse` command, build the callback
+    callable with the right signature.
+    (:issue:`6182`)
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Add a FAQ entry about :ref:`creating blank requests <faq-blank-request>`.
+    (:issue:`6203`, :issue:`6208`)
+
+-   Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``.
+    (:issue:`6328`, :issue:`6334`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Make builds reproducible. (:issue:`5019`, :issue:`6322`)
+
+-   Packaging and test fixes.
+    (:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`)
+
+
 .. _release-2.11.1:
 
 Scrapy 2.11.1 (2024-02-14)

docs/topics/benchmarking.rst

@@ -24,7 +24,8 @@ You should see an output like this::
      'scrapy.extensions.telnet.TelnetConsole',
      'scrapy.extensions.corestats.CoreStats']
     2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares:
-    ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
+    ['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',
+     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
      'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
      'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
      'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
@@ -37,7 +38,6 @@ You should see an output like this::
      'scrapy.downloadermiddlewares.stats.DownloaderStats']
     2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares:
     ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
-     'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
      'scrapy.spidermiddlewares.referer.RefererMiddleware',
      'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
      'scrapy.spidermiddlewares.depth.DepthMiddleware']

docs/topics/downloader-middleware.rst

@@ -763,6 +763,44 @@ HttpProxyMiddleware
    Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
    environment variables, and it will also ignore ``no_proxy`` environment variable.
 
+OffsiteMiddleware
+-----------------
+
+.. module:: scrapy.downloadermiddlewares.offsite
+   :synopsis: Offsite Middleware
+
+.. class:: OffsiteMiddleware
+
+   .. versionadded:: 2.11.2
+
+   Filters out Requests for URLs outside the domains covered by the spider.
+
+   This middleware filters out every request whose host names aren't in the
+   spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
+   All subdomains of any domain in the list are also allowed.
+   E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
+   but not ``www2.example.com`` nor ``example.com``.
+
+   When your spider returns a request for a domain not belonging to those
+   covered by the spider, this middleware will log a debug message similar to
+   this one::
+
+      DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
+
+   To avoid filling the log with too much noise, it will only print one of
+   these messages for each new domain filtered. So, for example, if another
+   request for ``offsite.example`` is filtered, no log message will be
+   printed. But if a request for ``other.example`` is filtered, a message
+   will be printed (but only for the first request filtered).
+
+   If the spider doesn't define an
+   :attr:`~scrapy.Spider.allowed_domains` attribute, or the
+   attribute is empty, the offsite middleware will allow all requests.
+
+   If the request has the :attr:`~scrapy.Request.dont_filter` attribute
+   set, the offsite middleware will allow the request even if its domain is not
+   listed in allowed domains.
+
 RedirectMiddleware
 ------------------
 
@@ -882,7 +920,11 @@ Meta tags within these tags are ignored.
 
 .. versionchanged:: 2.0
    The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
-   ``['script', 'noscript']`` to ``[]``.
+   ``["script", "noscript"]`` to ``[]``.
+
+.. versionchanged:: 2.11.2
+   The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
+   ``[]`` to ``["noscript"]``.
 
 .. setting:: METAREFRESH_MAXDELAY
 

docs/topics/feed-exports.rst

@@ -390,7 +390,13 @@ Each plugin is a class that must implement the following methods:
 
 .. method:: close(self)
 
-    Close the target file object.
+    Clean up the plugin.
+
+    For example, you might want to close a file wrapper that you might have
+    used to compress data written into the file received in the ``__init__``
+    method.
+
+    .. warning:: Do not close the file from the ``__init__`` method.
 
 To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
 can then access those parameters from the ``__init__`` method of your plugin.

docs/topics/practices.rst

@@ -289,7 +289,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
   services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
   super proxy that you can attach your own proxies to.
 * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
-  plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__
+  plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional 
+  features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__
 
 If you are still unable to prevent your bot getting banned, consider contacting
 `commercial support`_.

docs/topics/settings.rst

@@ -674,6 +674,7 @@ Default:
 .. code-block:: python
 
     {
+        "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,
         "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
         "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
         "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
@@ -1122,7 +1123,7 @@ modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
 JOBDIR
 ------
 
-Default: ``''``
+Default: ``None``
 
 A string indicating the directory for storing the state of a crawl when
 :ref:`pausing and resuming crawls <topics-jobs>`.
@@ -1605,7 +1606,6 @@ Default:
 
     {
         "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
-        "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
         "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
         "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
         "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,

docs/topics/signals.rst

@@ -343,11 +343,18 @@ request_scheduled
 .. signal:: request_scheduled
 .. function:: request_scheduled(request, spider)
 
-    Sent when the engine schedules a :class:`~scrapy.Request`, to be
+    Sent when the engine is asked to schedule a :class:`~scrapy.Request`, to be
-    downloaded later.
+    downloaded later, before the request reaches the :ref:`scheduler
+    <topics-scheduler>`.
+
+    Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it
+    reaches the scheduler.
 
     This signal does not support returning deferreds from its handlers.
 
+    .. versionadded:: 2.11.2
+        Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`.
+
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.Request` object
 

docs/topics/spider-middleware.rst

@@ -51,8 +51,8 @@ value.  For example, if you want to disable the off-site middleware:
 .. code-block:: python
 
     SPIDER_MIDDLEWARES = {
-        "myproject.middlewares.CustomSpiderMiddleware": 543,
+        "scrapy.spidermiddlewares.referer.RefererMiddleware": None,
-        "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
+        "myproject.middlewares.CustomRefererSpiderMiddleware": 700,
     }
 
 Finally, keep in mind that some middlewares may need to be enabled through a
@@ -313,42 +313,6 @@ Default: ``False``
 
 Pass all responses, regardless of its status code.
 
-OffsiteMiddleware
------------------
-
-.. module:: scrapy.spidermiddlewares.offsite
-   :synopsis: Offsite Spider Middleware
-
-.. class:: OffsiteMiddleware
-
-   Filters out Requests for URLs outside the domains covered by the spider.
-
-   This middleware filters out every request whose host names aren't in the
-   spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
-   All subdomains of any domain in the list are also allowed.
-   E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
-   but not ``www2.example.com`` nor ``example.com``.
-
-   When your spider returns a request for a domain not belonging to those
-   covered by the spider, this middleware will log a debug message similar to
-   this one::
-
-      DEBUG: Filtered offsite request to 'www.othersite.com': <GET http://www.othersite.com/some/page.html>
-
-   To avoid filling the log with too much noise, it will only print one of
-   these messages for each new domain filtered. So, for example, if another
-   request for ``www.othersite.com`` is filtered, no log message will be
-   printed. But if a request for ``someothersite.com`` is filtered, a message
-   will be printed (but only for the first request filtered).
-
-   If the spider doesn't define an
-   :attr:`~scrapy.Spider.allowed_domains` attribute, or the
-   attribute is empty, the offsite middleware will allow all requests.
-
-   If the request has the :attr:`~scrapy.Request.dont_filter` attribute
-   set, the offsite middleware will allow the request even if its domain is not
-   listed in allowed domains.
-
 
 RefererMiddleware
 -----------------

docs/topics/spiders.rst

@@ -75,7 +75,8 @@ scrapy.Spider
        An optional list of strings containing domains that this spider is
        allowed to crawl. Requests for URLs not belonging to the domain names
        specified in this list (or their subdomains) won't be followed if
-       :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
+       :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is
+       enabled.
 
        Let's say your target url is ``https://www.example.com/1.html``,
        then add ``'example.com'`` to the list.

scrapy/VERSION

@@ -1 +1 @@
-2.11.1
+2.11.2

scrapy/commands/parse.py

@@ -1,3 +1,4 @@
+import functools
 import inspect
 import json
 import logging
@@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand):
 
         return scraped_data
 
+    def _get_callback(self, *, spider, opts, response=None):
+        cb = None
+        if response:
+            cb = response.meta["_callback"]
+        if not cb:
+            if opts.callback:
+                cb = opts.callback
+            elif response and opts.rules and self.first_response == response:
+                cb = self.get_callback_from_rules(spider, response)
+                if not cb:
+                    raise ValueError(
+                        f"Cannot find a rule that matches {response.url!r} in spider: "
+                        f"{spider.name}"
+                    )
+            else:
+                cb = "parse"
+
+        if not callable(cb):
+            cb_method = getattr(spider, cb, None)
+            if callable(cb_method):
+                cb = cb_method
+            else:
+                raise ValueError(
+                    f"Cannot find callback {cb!r} in spider: {spider.name}"
+                )
+        return cb
+
     def prepare_request(self, spider, request, opts):
         def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
 
-            # determine real callback
+            cb = self._get_callback(spider=spider, opts=opts, response=response)
-            cb = response.meta["_callback"]
-            if not cb:
-                if opts.callback:
-                    cb = opts.callback
-                elif opts.rules and self.first_response == response:
-                    cb = self.get_callback_from_rules(spider, response)
-
-                    if not cb:
-                        logger.error(
-                            "Cannot find a rule that matches %(url)r in spider: %(spider)s",
-                            {"url": response.url, "spider": spider.name},
-                        )
-                        return
-                else:
-                    cb = "parse"
-
-            if not callable(cb):
-                cb_method = getattr(spider, cb, None)
-                if callable(cb_method):
-                    cb = cb_method
-                else:
-                    logger.error(
-                        "Cannot find callback %(callback)r in spider: %(spider)s",
-                        {"callback": cb, "spider": spider.name},
-                    )
-                    return
 
             # parse items and requests
             depth = response.meta["_depth"]
@@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand):
 
         request.meta["_depth"] = 1
         request.meta["_callback"] = request.callback
+        if not request.callback and not opts.rules:
+            cb = self._get_callback(spider=spider, opts=opts)
+            functools.update_wrapper(callback, cb)
         request.callback = callback
         return request
 

scrapy/core/engine.py

@@ -27,7 +27,7 @@ from twisted.python.failure import Failure
 from scrapy import signals
 from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import CloseSpider, DontCloseSpider
+from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
 from scrapy.http import Request, Response
 from scrapy.logformatter import LogFormatter
 from scrapy.settings import BaseSettings, Settings
@@ -35,6 +35,7 @@ from scrapy.signalmanager import SignalManager
 from scrapy.spiders import Spider
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.python import global_object_name
 from scrapy.utils.reactor import CallLaterOnce
 
 if TYPE_CHECKING:
@@ -291,9 +292,19 @@ class ExecutionEngine:
         self.slot.nextcall.schedule()  # type: ignore[union-attr]
 
     def _schedule_request(self, request: Request, spider: Spider) -> None:
-        self.signals.send_catch_log(
+        request_scheduled_result = self.signals.send_catch_log(
-            signals.request_scheduled, request=request, spider=spider
+            signals.request_scheduled,
+            request=request,
+            spider=spider,
+            dont_log=IgnoreRequest,
         )
+        for handler, result in request_scheduled_result:
+            if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest):
+                logger.debug(
+                    f"Signal handler {global_object_name(handler)} dropped "
+                    f"request {request} before it reached the scheduler."
+                )
+                return
         if not self.slot.scheduler.enqueue_request(request):  # type: ignore[union-attr]
             self.signals.send_catch_log(
                 signals.request_dropped, request=request, spider=spider

scrapy/core/scheduler.py

@@ -352,7 +352,7 @@ class Scheduler(BaseScheduler):
 
     def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
         """Return a folder name to keep disk queue state at"""
-        if jobdir is not None:
+        if jobdir:
             dqdir = Path(jobdir, "requests.queue")
             if not dqdir.exists():
                 dqdir.mkdir(parents=True)

scrapy/downloadermiddlewares/httpproxy.py

@@ -44,23 +44,26 @@ class HttpProxyMiddleware:
         return creds, proxy_url
 
     def process_request(self, request, spider):
-        creds, proxy_url = None, None
+        creds, proxy_url, scheme = None, None, None
         if "proxy" in request.meta:
             if request.meta["proxy"] is not None:
                 creds, proxy_url = self._get_proxy(request.meta["proxy"], "")
         elif self.proxies:
             parsed = urlparse_cached(request)
-            scheme = parsed.scheme
+            _scheme = parsed.scheme
             if (
                 # 'no_proxy' is only supported by http schemes
-                scheme not in ("http", "https")
+                _scheme not in ("http", "https")
                 or not proxy_bypass(parsed.hostname)
-            ) and scheme in self.proxies:
+            ) and _scheme in self.proxies:
+                scheme = _scheme
                 creds, proxy_url = self.proxies[scheme]
 
-        self._set_proxy_and_creds(request, proxy_url, creds)
+        self._set_proxy_and_creds(request, proxy_url, creds, scheme)
 
-    def _set_proxy_and_creds(self, request, proxy_url, creds):
+    def _set_proxy_and_creds(self, request, proxy_url, creds, scheme):
+        if scheme:
+            request.meta["_scheme_proxy"] = True
         if proxy_url:
             request.meta["proxy"] = proxy_url
         elif request.meta.get("proxy") is not None:

scrapy/downloadermiddlewares/offsite.py

@@ -0,0 +1,77 @@
+import logging
+import re
+import warnings
+
+from scrapy import signals
+from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.httpobj import urlparse_cached
+
+logger = logging.getLogger(__name__)
+
+
+class OffsiteMiddleware:
+    @classmethod
+    def from_crawler(cls, crawler):
+        o = cls(crawler.stats)
+        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+        crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
+        return o
+
+    def __init__(self, stats):
+        self.stats = stats
+        self.domains_seen = set()
+
+    def spider_opened(self, spider):
+        self.host_regex = self.get_host_regex(spider)
+
+    def request_scheduled(self, request, spider):
+        self.process_request(request, spider)
+
+    def process_request(self, request, spider):
+        if request.dont_filter or self.should_follow(request, spider):
+            return None
+        domain = urlparse_cached(request).hostname
+        if domain and domain not in self.domains_seen:
+            self.domains_seen.add(domain)
+            logger.debug(
+                "Filtered offsite request to %(domain)r: %(request)s",
+                {"domain": domain, "request": request},
+                extra={"spider": spider},
+            )
+            self.stats.inc_value("offsite/domains", spider=spider)
+        self.stats.inc_value("offsite/filtered", spider=spider)
+        raise IgnoreRequest
+
+    def should_follow(self, request, spider):
+        regex = self.host_regex
+        # hostname can be None for wrong urls (like javascript links)
+        host = urlparse_cached(request).hostname or ""
+        return bool(regex.search(host))
+
+    def get_host_regex(self, spider):
+        """Override this method to implement a different offsite policy"""
+        allowed_domains = getattr(spider, "allowed_domains", None)
+        if not allowed_domains:
+            return re.compile("")  # allow all by default
+        url_pattern = re.compile(r"^https?://.*$")
+        port_pattern = re.compile(r":\d+$")
+        domains = []
+        for domain in allowed_domains:
+            if domain is None:
+                continue
+            if url_pattern.match(domain):
+                message = (
+                    "allowed_domains accepts only domains, not URLs. "
+                    f"Ignoring URL entry {domain} in allowed_domains."
+                )
+                warnings.warn(message)
+            elif port_pattern.search(domain):
+                message = (
+                    "allowed_domains accepts only domains without ports. "
+                    f"Ignoring entry {domain} in allowed_domains."
+                )
+                warnings.warn(message)
+            else:
+                domains.append(re.escape(domain))
+        regex = rf'^(.*\.)?({"|".join(domains)})$'
+        return re.compile(regex)

scrapy/downloadermiddlewares/redirect.py

@@ -17,17 +17,49 @@ def _build_redirect_request(source_request, *, url, **kwargs):
         **kwargs,
         cookies=None,
     )
+    if "_scheme_proxy" in redirect_request.meta:
+        source_request_scheme = urlparse_cached(source_request).scheme
+        redirect_request_scheme = urlparse_cached(redirect_request).scheme
+        if source_request_scheme != redirect_request_scheme:
+            redirect_request.meta.pop("_scheme_proxy")
+            redirect_request.meta.pop("proxy", None)
+            redirect_request.meta.pop("_auth_proxy", None)
+            redirect_request.headers.pop(b"Proxy-Authorization", None)
     has_cookie_header = "Cookie" in redirect_request.headers
     has_authorization_header = "Authorization" in redirect_request.headers
     if has_cookie_header or has_authorization_header:
-        source_request_netloc = urlparse_cached(source_request).netloc
+        default_ports = {"http": 80, "https": 443}
-        redirect_request_netloc = urlparse_cached(redirect_request).netloc
+
-        if source_request_netloc != redirect_request_netloc:
+        parsed_source_request = urlparse_cached(source_request)
-            if has_cookie_header:
+        source_scheme, source_host, source_port = (
-                del redirect_request.headers["Cookie"]
+            parsed_source_request.scheme,
-            # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name
+            parsed_source_request.hostname,
-            if has_authorization_header:
+            parsed_source_request.port
-                del redirect_request.headers["Authorization"]
+            or default_ports.get(parsed_source_request.scheme),
+        )
+
+        parsed_redirect_request = urlparse_cached(redirect_request)
+        redirect_scheme, redirect_host, redirect_port = (
+            parsed_redirect_request.scheme,
+            parsed_redirect_request.hostname,
+            parsed_redirect_request.port
+            or default_ports.get(parsed_redirect_request.scheme),
+        )
+
+        if has_cookie_header and (
+            (source_scheme != redirect_scheme and redirect_scheme != "https")
+            or source_host != redirect_host
+        ):
+            del redirect_request.headers["Cookie"]
+
+        # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name
+        if has_authorization_header and (
+            source_scheme != redirect_scheme
+            or source_host != redirect_host
+            or source_port != redirect_port
+        ):
+            del redirect_request.headers["Authorization"]
+
     return redirect_request
 
 
@@ -110,6 +142,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
             location = request_scheme + "://" + location.lstrip("/")
 
         redirected_url = urljoin(request.url, location)
+        if urlparse(redirected_url).scheme not in {"http", "https"}:
+            return response
 
         if response.status in (301, 307, 308) or request.method == "HEAD":
             redirected = _build_redirect_request(request, url=redirected_url)
@@ -132,12 +166,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
             request.meta.get("dont_redirect", False)
             or request.method == "HEAD"
             or not isinstance(response, HtmlResponse)
+            or urlparse_cached(request).scheme not in {"http", "https"}
         ):
             return response
 
         interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
-        if url and interval < self._maxdelay:
+        if not url:
+            return response
+        if urlparse(url).scheme not in {"http", "https"}:
+            return response
+        if interval < self._maxdelay:
             redirected = self._redirect_request_using_get(request, url)
             return self._redirect(redirected, request, spider, "meta refresh")
-
         return response

scrapy/extensions/memusage.py

@@ -128,9 +128,9 @@ class MemoryUsage:
     def _send_report(self, rcpts, subject):
         """send notification mail with some additional useful info"""
         stats = self.crawler.stats
-        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
+        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
-        s += f"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\r\n"
+        s += f"Maximum memory usage          : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
-        s += f"Current memory usage          : {self.get_virtual_size()/1024/1024}M\r\n"
+        s += f"Current memory usage          : {self.get_virtual_size() / 1024 / 1024}M\r\n"
 
         s += (
             "ENGINE STATUS ------------------------------------------------------- \r\n"

scrapy/extensions/postprocessing.py

@@ -42,7 +42,6 @@ class GzipPlugin:
 
     def close(self) -> None:
         self.gzipfile.close()
-        self.file.close()
 
 
 class Bz2Plugin:
@@ -69,7 +68,6 @@ class Bz2Plugin:
 
     def close(self) -> None:
         self.bz2file.close()
-        self.file.close()
 
 
 class LZMAPlugin:
@@ -111,7 +109,6 @@ class LZMAPlugin:
 
     def close(self) -> None:
         self.lzmafile.close()
-        self.file.close()
 
 
 # io.IOBase is subclassed here, so that exporters can use the PostProcessingManager

scrapy/http/cookies.py

@@ -166,7 +166,8 @@ class WrappedRequest:
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_unicode(self.request.headers.get(name, default), errors="replace")
+        value = self.request.headers.get(name, default)
+        return to_unicode(value, errors="replace") if value is not None else None
 
     def header_items(self):
         return [

scrapy/http/request/form.py

@@ -8,21 +8,16 @@ See documentation in docs/topics/request-response.rst
 from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
-from lxml.html import (
+from lxml.html import FormElement  # nosec
-    FormElement,
+from lxml.html import InputElement  # nosec
-    HTMLParser,
+from lxml.html import MultipleSelectOptions  # nosec
-    InputElement,
+from lxml.html import SelectElement  # nosec
-    MultipleSelectOptions,
+from lxml.html import TextareaElement  # nosec
-    SelectElement,
-    TextareaElement,
-)
-from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
 from scrapy.http.response.text import TextResponse
 from scrapy.utils.python import is_listlike, to_bytes
-from scrapy.utils.response import get_base_url
 
 FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
 
@@ -113,7 +108,7 @@ def _get_form(
     formxpath: Optional[str],
 ) -> FormElement:
     """Find the wanted form element within the given response."""
-    root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
+    root = response.selector.root
     forms = root.xpath("//form")
     if not forms:
         raise ValueError(f"No <form> element found in {response}")

scrapy/http/request/rpc.py

@@ -7,9 +7,13 @@ See documentation in docs/topics/request-response.rst
 import xmlrpc.client as xmlrpclib
 from typing import Optional
 
+import defusedxml.xmlrpc
+
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
 
+defusedxml.xmlrpc.monkey_patch()
+
 DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 

scrapy/linkextractors/lxmlhtml.py

@@ -6,7 +6,7 @@ import operator
 from functools import partial
 from urllib.parse import urljoin, urlparse
 
-from lxml import etree
+from lxml import etree  # nosec
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
@@ -153,7 +153,7 @@ class LxmlLinkExtractor:
             unique=unique,
             process=process_value,
             strip=strip,
-            canonicalized=canonicalize,
+            canonicalized=not canonicalize,
         )
         self.allow_res = [
             x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)
@@ -248,5 +248,5 @@ class LxmlLinkExtractor:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
         if self.link_extractor.unique:
-            return unique_list(all_links)
+            return unique_list(all_links, key=self.link_extractor.link_key)
         return all_links

scrapy/pipelines/media.py

@@ -118,14 +118,14 @@ class MediaPipeline:
         info.downloading.add(fp)
         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
         dfd.addCallback(self._check_media_to_download, request, info, item=item)
+        dfd.addErrback(self._log_exception)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        dfd.addErrback(
-            lambda f: logger.error(
-                f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
-            )
-        )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _log_exception(self, result):
+        logger.exception(result)
+        return result
+
     def _make_compatible(self):
         """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
         methods = [

scrapy/robotstxt.py

@@ -12,7 +12,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         if to_native_str_type:
             robotstxt_body = to_unicode(robotstxt_body)
         else:
-            robotstxt_body = robotstxt_body.decode("utf-8")
+            robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.

scrapy/selector/unified.py

@@ -7,6 +7,7 @@ from parsel import Selector as _ParselSelector
 
 from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.utils.python import to_bytes
+from scrapy.utils.response import get_base_url
 from scrapy.utils.trackref import object_ref
 
 __all__ = ["Selector", "SelectorList"]
@@ -45,7 +46,7 @@ class Selector(_ParselSelector, object_ref):
     ``response`` isn't available. Using ``text`` and ``response`` together is
     undefined behavior.
 
-    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"``
     or ``None`` (default).
 
     If ``type`` is ``None``, the selector automatically chooses the best type
@@ -87,7 +88,7 @@ class Selector(_ParselSelector, object_ref):
 
         if response is not None:
             text = response.text
-            kwargs.setdefault("base_url", response.url)
+            kwargs.setdefault("base_url", get_base_url(response))
 
         self.response = response
 

scrapy/settings/default_settings.py

@@ -101,6 +101,7 @@ DOWNLOADER_MIDDLEWARES = {}
 
 DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
+    "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,
     "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
     "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
     "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
@@ -206,6 +207,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
 ITEM_PIPELINES = {}
 ITEM_PIPELINES_BASE = {}
 
+JOBDIR = None
+
 LOG_ENABLED = True
 LOG_ENCODING = "utf-8"
 LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
@@ -237,7 +240,7 @@ MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True
-METAREFRESH_IGNORE_TAGS = []
+METAREFRESH_IGNORE_TAGS = ["noscript"]
 METAREFRESH_MAXDELAY = 100
 
 NEWSPIDER_MODULE = ""
@@ -299,7 +302,6 @@ SPIDER_MIDDLEWARES = {}
 SPIDER_MIDDLEWARES_BASE = {
     # Engine side
     "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
-    "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
     "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
     "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
     "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,

scrapy/spidermiddlewares/offsite.py

@@ -8,9 +8,16 @@ import re
 import warnings
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
+warnings.warn(
+    "The scrapy.spidermiddlewares.offsite module is deprecated, use "
+    "scrapy.downloadermiddlewares.offsite instead.",
+    ScrapyDeprecationWarning,
+)
+
 logger = logging.getLogger(__name__)
 
 

scrapy/spiders/__init__.py

@@ -21,6 +21,7 @@ if TYPE_CHECKING:
 
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
+    from scrapy.utils.log import SpiderLoggerAdapter
 
 
 class Spider(object_ref):
@@ -41,9 +42,11 @@ class Spider(object_ref):
             self.start_urls: List[str] = []
 
     @property
-    def logger(self) -> logging.LoggerAdapter:
+    def logger(self) -> SpiderLoggerAdapter:
+        from scrapy.utils.log import SpiderLoggerAdapter
+
         logger = logging.getLogger(self.name)
-        return logging.LoggerAdapter(logger, {"spider": self})
+        return SpiderLoggerAdapter(logger, {"spider": self})
 
     def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:
         """Log the given message at the given log level

scrapy/utils/_compression.py

@@ -1,10 +1,39 @@
 import zlib
 from io import BytesIO
+from warnings import warn
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
     import brotli
 except ImportError:
     pass
+else:
+    try:
+        brotli.Decompressor.process
+    except AttributeError:
+        warn(
+            (
+                "You have brotlipy installed, and Scrapy will use it, but "
+                "Scrapy support for brotlipy is deprecated and will stop "
+                "working in a future version of Scrapy. brotlipy itself is "
+                "deprecated, it has been superseded by brotlicffi (not "
+                "currently supported by Scrapy). Please, uninstall brotlipy "
+                "and install brotli instead. brotlipy has the same import "
+                "name as brotli, so keeping both installed is strongly "
+                "discouraged."
+            ),
+            ScrapyDeprecationWarning,
+        )
+
+        def _brotli_decompress(decompressor, data):
+            return decompressor.decompress(data)
+
+    else:
+
+        def _brotli_decompress(decompressor, data):
+            return decompressor.process(data)
+
 
 try:
     import zstandard
@@ -61,7 +90,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
     decompressed_size = 0
     while output_chunk:
         input_chunk = input_stream.read(_CHUNK_SIZE)
-        output_chunk = decompressor.process(input_chunk)
+        output_chunk = _brotli_decompress(decompressor, input_chunk)
         decompressed_size += len(output_chunk)
         if max_size and decompressed_size > max_size:
             raise _DecompressionMaxSizeExceeded(

scrapy/utils/iterators.py

@@ -18,7 +18,7 @@ from typing import (
 )
 from warnings import warn
 
-from lxml import etree
+from lxml import etree  # nosec
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
@@ -26,7 +26,7 @@ from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
 if TYPE_CHECKING:
-    from lxml._types import SupportsReadClose
+    from lxml._types import SupportsReadClose  # nosec
 
 logger = logging.getLogger(__name__)
 
@@ -101,6 +101,7 @@ def xmliter_lxml(
         cast("SupportsReadClose[bytes]", reader),
         encoding=reader.encoding,
         events=("end", "start-ns"),
+        resolve_entities=False,
         huge_tree=True,
     )
     selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)

scrapy/utils/job.py

@@ -5,7 +5,9 @@ from scrapy.settings import BaseSettings
 
 
 def job_dir(settings: BaseSettings) -> Optional[str]:
-    path: str = settings["JOBDIR"]
+    path: Optional[str] = settings["JOBDIR"]
-    if path and not Path(path).exists():
+    if not path:
+        return None
+    if not Path(path).exists():
         Path(path).mkdir(parents=True)
     return path

scrapy/utils/log.py

@@ -5,7 +5,17 @@ import sys
 import warnings
 from logging.config import dictConfig
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    List,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -249,3 +259,16 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
     args = logkws if not logkws.get("args") else logkws["args"]
 
     return (level, message, args)
+
+
+class SpiderLoggerAdapter(logging.LoggerAdapter):
+    def process(
+        self, msg: str, kwargs: MutableMapping[str, Any]
+    ) -> Tuple[str, MutableMapping[str, Any]]:
+        """Method that augments logging with additional 'extra' data"""
+        if isinstance(kwargs.get("extra"), MutableMapping):
+            kwargs["extra"].update(self.extra)
+        else:
+            kwargs["extra"] = self.extra
+
+        return msg, kwargs

scrapy/utils/python.py

@@ -316,7 +316,7 @@ def global_object_name(obj: Any) -> str:
     >>> global_object_name(Request)
     'scrapy.http.request.Request'
     """
-    return f"{obj.__module__}.{obj.__name__}"
+    return f"{obj.__module__}.{obj.__qualname__}"
 
 
 if hasattr(sys, "pypy_version_info"):

scrapy/utils/sitemap.py

@@ -7,7 +7,7 @@ SitemapSpider, its API is subject to change without notice.
 from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 
-import lxml.etree
+import lxml.etree  # nosec
 
 
 class Sitemap:
@@ -18,7 +18,7 @@ class Sitemap:
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )
-        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
+        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)  # nosec
         rt = self._root.tag
         self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
 

scrapy/utils/versions.py

@@ -4,7 +4,7 @@ from typing import List, Tuple
 
 import cryptography
 import cssselect
-import lxml.etree
+import lxml.etree  # nosec
 import parsel
 import twisted
 import w3lib

setup.py

@@ -22,6 +22,7 @@ install_requires = [
     "packaging",
     "tldextract",
     "lxml>=4.4.1",
+    "defusedxml>=0.7.1",
 ]
 extras_require = {
     ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],

tests/spiders.py

@@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking
 """
 import asyncio
 import time
+from typing import Optional
 from urllib.parse import urlencode
 
 from twisted.internet import defer
@@ -77,6 +78,28 @@ class DelaySpider(MetaSpider):
         self.t2_err = time.time()
 
 
+class LogSpider(MetaSpider):
+    name = "log_spider"
+
+    def log_debug(self, message: str, extra: Optional[dict] = None):
+        self.logger.debug(message, extra=extra)
+
+    def log_info(self, message: str, extra: Optional[dict] = None):
+        self.logger.info(message, extra=extra)
+
+    def log_warning(self, message: str, extra: Optional[dict] = None):
+        self.logger.warning(message, extra=extra)
+
+    def log_error(self, message: str, extra: Optional[dict] = None):
+        self.logger.error(message, extra=extra)
+
+    def log_critical(self, message: str, extra: Optional[dict] = None):
+        self.logger.critical(message, extra=extra)
+
+    def parse(self, response):
+        pass
+
+
 class SlowSpider(DelaySpider):
     name = "slow"
 

tests/test_command_check.py

@@ -16,11 +16,11 @@ import scrapy
 
 class CheckSpider(scrapy.Spider):
     name = '{self.spider_name}'
-    start_urls = ['http://toscrape.com']
+    start_urls = ['data:,']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://toscrape.com
+        @url data:,
         {contracts}
         \"\"\"
         {parse_def}

tests/test_command_parse.py

@@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
             if i > 5:
                 raise ValueError("Stopping the processing")
 
+class CallbackSignatureDownloaderMiddleware:
+    def process_request(self, request, spider):
+        from inspect import signature
+        spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}")
+
+
 class MySpider(scrapy.Spider):
     name = '{self.spider_name}'
 
+    custom_settings = {{
+        "DOWNLOADER_MIDDLEWARES": {{
+            CallbackSignatureDownloaderMiddleware: 0,
+        }}
+    }}
+
     def parse(self, response):
         if getattr(self, 'test_arg', None):
             self.logger.debug('It Works!')
@@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
                 self.url("/html"),
             ]
         )
-        self.assertIn("DEBUG: It Works!", _textmode(stderr))
+        log = _textmode(stderr)
+        self.assertIn("DEBUG: It Works!", log)
+        self.assertIn(
+            "DEBUG: request.callback signature: (response, foo=None, key=None)", log
+        )
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):

tests/test_commands.py

@@ -988,12 +988,13 @@ class MySpider(scrapy.Spider):
         self.assertIn("The value of FOO is 42", log)
 
 
-@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
     spider_filename = "myspider.pyw"
 
     def setUp(self):
-        super().setUp()
+        if platform.system() != "Windows":
+            raise unittest.SkipTest("Windows required for .pyw files")
+        return super().setUp()
 
     def test_start_requests_errors(self):
         log = self.get_log(self.badspider, name="badspider.pyw")

tests/test_downloadermiddleware.py

@@ -22,13 +22,11 @@ class ManagerTestCase(TestCase):
         self.crawler = get_crawler(Spider, self.settings_dict)
         self.spider = self.crawler._create_spider("foo")
         self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler)
-        # some mw depends on stats collector
+        self.crawler.engine = self.crawler._create_engine()
-        self.crawler.stats.open_spider(self.spider)
+        return self.crawler.engine.open_spider(self.spider, start_requests=())
-        return self.mwman.open_spider(self.spider)
 
     def tearDown(self):
-        self.crawler.stats.close_spider(self.spider, "")
+        return self.crawler.engine.close_spider(self.spider)
-        return self.mwman.close_spider(self.spider)
 
     def _download(self, request, response=None):
         """Executes downloader mw manager's download method and returns

tests/test_downloadermiddleware_offsite.py

@@ -0,0 +1,184 @@
+import pytest
+
+from scrapy import Request, Spider
+from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
+from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.test import get_crawler
+
+UNSET = object()
+
+
+@pytest.mark.parametrize(
+    ("allowed_domain", "url", "allowed"),
+    (
+        ("example.com", "http://example.com/1", True),
+        ("example.com", "http://example.org/1", False),
+        ("example.com", "http://sub.example.com/1", True),
+        ("sub.example.com", "http://sub.example.com/1", True),
+        ("sub.example.com", "http://example.com/1", False),
+        ("example.com", "http://example.com:8000/1", True),
+        ("example.com", "http://example.org/example.com", False),
+        ("example.com", "http://example.org/foo.example.com", False),
+        ("example.com", "http://example.com.example", False),
+        ("a.example", "http://nota.example", False),
+        ("b.a.example", "http://notb.a.example", False),
+    ),
+)
+def test_process_request_domain_filtering(allowed_domain, url, allowed):
+    crawler = get_crawler(Spider)
+    spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    request = Request(url)
+    if allowed:
+        assert mw.process_request(request, spider) is None
+    else:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request, spider)
+
+
+@pytest.mark.parametrize(
+    ("value", "filtered"),
+    (
+        (UNSET, True),
+        (None, True),
+        (False, True),
+        (True, False),
+    ),
+)
+def test_process_request_dont_filter(value, filtered):
+    crawler = get_crawler(Spider)
+    spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["dont_filter"] = value
+    request = Request("https://b.example", **kwargs)
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request, spider)
+    else:
+        assert mw.process_request(request, spider) is None
+
+
+@pytest.mark.parametrize(
+    "value",
+    (
+        UNSET,
+        None,
+        [],
+    ),
+)
+def test_process_request_no_allowed_domains(value):
+    crawler = get_crawler(Spider)
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["allowed_domains"] = value
+    spider = crawler._create_spider(name="a", **kwargs)
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    request = Request("https://example.com")
+    assert mw.process_request(request, spider) is None
+
+
+def test_process_request_invalid_domains():
+    crawler = get_crawler(Spider)
+    allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
+    spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    request = Request("https://a.example")
+    assert mw.process_request(request, spider) is None
+    for letter in ("b", "c"):
+        request = Request(f"https://{letter}.example")
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request, spider)
+
+
+@pytest.mark.parametrize(
+    ("allowed_domain", "url", "allowed"),
+    (
+        ("example.com", "http://example.com/1", True),
+        ("example.com", "http://example.org/1", False),
+        ("example.com", "http://sub.example.com/1", True),
+        ("sub.example.com", "http://sub.example.com/1", True),
+        ("sub.example.com", "http://example.com/1", False),
+        ("example.com", "http://example.com:8000/1", True),
+        ("example.com", "http://example.org/example.com", False),
+        ("example.com", "http://example.org/foo.example.com", False),
+        ("example.com", "http://example.com.example", False),
+        ("a.example", "http://nota.example", False),
+        ("b.a.example", "http://notb.a.example", False),
+    ),
+)
+def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
+    crawler = get_crawler(Spider)
+    spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    request = Request(url)
+    if allowed:
+        assert mw.request_scheduled(request, spider) is None
+    else:
+        with pytest.raises(IgnoreRequest):
+            mw.request_scheduled(request, spider)
+
+
+@pytest.mark.parametrize(
+    ("value", "filtered"),
+    (
+        (UNSET, True),
+        (None, True),
+        (False, True),
+        (True, False),
+    ),
+)
+def test_request_scheduled_dont_filter(value, filtered):
+    crawler = get_crawler(Spider)
+    spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["dont_filter"] = value
+    request = Request("https://b.example", **kwargs)
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.request_scheduled(request, spider)
+    else:
+        assert mw.request_scheduled(request, spider) is None
+
+
+@pytest.mark.parametrize(
+    "value",
+    (
+        UNSET,
+        None,
+        [],
+    ),
+)
+def test_request_scheduled_no_allowed_domains(value):
+    crawler = get_crawler(Spider)
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["allowed_domains"] = value
+    spider = crawler._create_spider(name="a", **kwargs)
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    request = Request("https://example.com")
+    assert mw.request_scheduled(request, spider) is None
+
+
+def test_request_scheduled_invalid_domains():
+    crawler = get_crawler(Spider)
+    allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
+    spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+    request = Request("https://a.example")
+    assert mw.request_scheduled(request, spider) is None
+    for letter in ("b", "c"):
+        request = Request(f"https://{letter}.example")
+        with pytest.raises(IgnoreRequest):
+            mw.request_scheduled(request, spider)

tests/test_engine.py

@@ -15,8 +15,10 @@ import subprocess
 import sys
 from collections import defaultdict
 from dataclasses import dataclass
+from logging import DEBUG
 from pathlib import Path
 from threading import Timer
+from unittest.mock import Mock
 from urllib.parse import urlparse
 
 import attr
@@ -27,11 +29,13 @@ from twisted.trial import unittest
 from twisted.web import server, static, util
 
 from scrapy import signals
-from scrapy.core.engine import ExecutionEngine
+from scrapy.core.engine import ExecutionEngine, Slot
-from scrapy.exceptions import CloseSpider
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.exceptions import CloseSpider, IgnoreRequest
 from scrapy.http import Request
 from scrapy.item import Field, Item
 from scrapy.linkextractors import LinkExtractor
+from scrapy.signals import request_scheduled
 from scrapy.spiders import Spider
 from scrapy.utils.signal import disconnect_all
 from scrapy.utils.test import get_crawler
@@ -467,6 +471,38 @@ class EngineTest(unittest.TestCase):
         self.assertNotIn(b"Traceback", stderr)
 
 
+def test_request_scheduled_signal(caplog):
+    class TestScheduler(BaseScheduler):
+        def __init__(self):
+            self.enqueued = []
+
+        def enqueue_request(self, request: Request) -> bool:
+            self.enqueued.append(request)
+            return True
+
+    def signal_handler(request: Request, spider: Spider) -> None:
+        if "drop" in request.url:
+            raise IgnoreRequest
+
+    spider = TestSpider()
+    crawler = get_crawler(spider.__class__)
+    engine = ExecutionEngine(crawler, lambda _: None)
+    engine.downloader._slot_gc_loop.stop()
+    scheduler = TestScheduler()
+    engine.slot = Slot((), None, Mock(), scheduler)
+    crawler.signals.connect(signal_handler, request_scheduled)
+    keep_request = Request("https://keep.example")
+    engine._schedule_request(keep_request, spider)
+    drop_request = Request("https://drop.example")
+    caplog.set_level(DEBUG)
+    engine._schedule_request(drop_request, spider)
+    assert scheduler.enqueued == [
+        keep_request
+    ], f"{scheduler.enqueued!r} != [{keep_request!r}]"
+    assert "dropped request <GET https://drop.example>" in caplog.text
+    crawler.signals.disconnect(signal_handler, request_scheduled)
+
+
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == "runserver":
         start_test_site(debug=True)

tests/test_feedexport.py

@@ -1732,6 +1732,7 @@ class FeedExportTest(FeedExportTestBase):
 
             def store(self, file):
                 Storage.store_file = file
+                Storage.file_was_closed = file.closed
                 file.close()
 
         settings = {
@@ -1747,6 +1748,7 @@ class FeedExportTest(FeedExportTestBase):
         }
         yield self.exported_no_data(settings)
         self.assertIs(Storage.open_file, Storage.store_file)
+        self.assertFalse(Storage.file_was_closed)
 
 
 class FeedPostProcessedExportsTest(FeedExportTestBase):

tests/test_http_cookies.py

@@ -43,6 +43,13 @@ class WrappedRequestTest(TestCase):
     def test_get_header(self):
         self.assertEqual(self.wrapped.get_header("content-type"), "text/html")
         self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def")
+        self.assertEqual(self.wrapped.get_header("xxxxx"), None)
+        wrapped = WrappedRequest(
+            Request(
+                "http://www.example.com/page.html", headers={"empty-binary-header": b""}
+            )
+        )
+        self.assertEqual(wrapped.get_header("empty-binary-header"), "")
 
     def test_header_items(self):
         self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])])

tests/test_linkextractors.py

@@ -744,6 +744,118 @@ class Base:
             lx = self.extractor_cls()
             self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
 
+        def test_link_extractor_aggregation(self):
+            """When a parameter like restrict_css is used, the underlying
+            implementation calls its internal link extractor once per selector
+            matching the specified restrictions, and then aggregates the
+            extracted links.
+
+            Test that aggregation respects the unique and canonicalize
+            parameters.
+            """
+            # unique=True (default), canonicalize=False (default)
+            lx = self.extractor_cls(restrict_css=("div",))
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                    Link(url="https://example.com/b?b=2&a=1", text="b2"),
+                ],
+            )
+
+            # unique=True (default), canonicalize=True
+            lx = self.extractor_cls(restrict_css=("div",), canonicalize=True)
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                ],
+            )
+
+            # unique=False, canonicalize=False (default)
+            lx = self.extractor_cls(restrict_css=("div",), unique=False)
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                    Link(url="https://example.com/a", text="a2"),
+                    Link(url="https://example.com/b?b=2&a=1", text="b2"),
+                ],
+            )
+
+            # unique=False, canonicalize=True
+            lx = self.extractor_cls(
+                restrict_css=("div",), unique=False, canonicalize=True
+            )
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                    Link(url="https://example.com/a", text="a2"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b2"),
+                ],
+            )
+
 
 class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor

tests/test_pipeline_crawl.py

@@ -8,6 +8,7 @@ from w3lib.url import add_or_replace_parameter
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner
+from scrapy.utils.misc import load_object
 from tests.mockserver import MockServer
 from tests.spiders import SimpleSpider
 
@@ -192,6 +193,29 @@ class FileDownloadCrawlTestCase(TestCase):
             crawler.stats.get_value("downloader/response_status_count/302"), 3
         )
 
+    @defer.inlineCallbacks
+    def test_download_media_file_path_error(self):
+        cls = load_object(self.pipeline_class)
+
+        class ExceptionRaisingMediaPipeline(cls):
+            def file_path(self, request, response=None, info=None, *, item=None):
+                return 1 / 0
+
+        settings = {
+            **self.settings,
+            "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
+        }
+        runner = CrawlerRunner(settings)
+        crawler = self._create_crawler(MediaDownloadSpider, runner=runner)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                self.mockserver.url("/files/images/"),
+                media_key=self.media_key,
+                media_urls_key=self.media_urls_key,
+                mockserver=self.mockserver,
+            )
+        self.assertIn("ZeroDivisionError", str(log))
+
 
 try:
     from PIL import Image  # noqa: imported just to check for the import error

tests/test_robotstxt_interface.py

@@ -1,5 +1,7 @@
 from twisted.trial import unittest
 
+from scrapy.robotstxt import decode_robotstxt
+
 
 def reppy_available():
     # check if reppy parser is installed
@@ -141,6 +143,25 @@ class BaseRobotParserTest:
         )
 
 
+class DecodeRobotsTxtTest(unittest.TestCase):
+    def test_native_string_conversion(self):
+        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        decoded_content = decode_robotstxt(
+            robotstxt_body, spider=None, to_native_str_type=True
+        )
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+    def test_decode_utf8(self):
+        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+    def test_decode_non_utf8(self):
+        robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n"
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser

tests/test_utils_iterators.py

@@ -546,6 +546,6 @@ class TestHelper(unittest.TestCase):
 
     def _assert_type_and_value(self, a, b, obj):
         self.assertTrue(
-            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
+            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
         )
         self.assertEqual(a, b)

tests/test_utils_log.py

@@ -1,18 +1,26 @@
+import json
 import logging
+import re
 import sys
 import unittest
+from io import StringIO
+from typing import Any, Dict, Mapping, MutableMapping
+from unittest import TestCase
 
+import pytest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy.extensions import telnet
 from scrapy.utils.log import (
     LogCounterHandler,
+    SpiderLoggerAdapter,
     StreamLogger,
     TopLevelFormatter,
     failure_to_exc_info,
 )
 from scrapy.utils.test import get_crawler
+from tests.spiders import LogSpider
 
 
 class FailureToExcInfoTest(unittest.TestCase):
@@ -106,3 +114,181 @@ class StreamLoggerTest(unittest.TestCase):
         with LogCapture() as log:
             print("test log msg")
         log.check(("test", "ERROR", "test log msg"))
+
+
+@pytest.mark.parametrize(
+    ("base_extra", "log_extra", "expected_extra"),
+    (
+        (
+            {"spider": "test"},
+            {"extra": {"log_extra": "info"}},
+            {"extra": {"log_extra": "info", "spider": "test"}},
+        ),
+        (
+            {"spider": "test"},
+            {"extra": None},
+            {"extra": {"spider": "test"}},
+        ),
+        (
+            {"spider": "test"},
+            {"extra": {"spider": "test2"}},
+            {"extra": {"spider": "test"}},
+        ),
+    ),
+)
+def test_spider_logger_adapter_process(
+    base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict
+):
+    logger = logging.getLogger("test")
+    spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
+
+    log_message = "test_log_message"
+    result_message, result_kwargs = spider_logger_adapter.process(
+        log_message, log_extra
+    )
+
+    assert result_message == log_message
+    assert result_kwargs == expected_extra
+
+
+class LoggingTestCase(TestCase):
+    def setUp(self):
+        self.log_stream = StringIO()
+        handler = logging.StreamHandler(self.log_stream)
+        logger = logging.getLogger("log_spider")
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+        self.handler = handler
+        self.logger = logger
+        self.spider = LogSpider()
+
+    def tearDown(self):
+        self.logger.removeHandler(self.handler)
+
+    def test_debug_logging(self):
+        log_message = "Foo message"
+        self.spider.log_debug(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_info_logging(self):
+        log_message = "Bar message"
+        self.spider.log_info(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_warning_logging(self):
+        log_message = "Baz message"
+        self.spider.log_warning(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_error_logging(self):
+        log_message = "Foo bar message"
+        self.spider.log_error(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_critical_logging(self):
+        log_message = "Foo bar baz message"
+        self.spider.log_critical(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+
+class LoggingWithExtraTestCase(TestCase):
+    def setUp(self):
+        self.log_stream = StringIO()
+        handler = logging.StreamHandler(self.log_stream)
+        formatter = logging.Formatter(
+            '{"levelname": "%(levelname)s", "message": "%(message)s", '
+            '"spider": "%(spider)s", "important_info": "%(important_info)s"}'
+        )
+        handler.setFormatter(formatter)
+        logger = logging.getLogger("log_spider")
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+        self.handler = handler
+        self.logger = logger
+        self.spider = LogSpider()
+        self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
+
+    def tearDown(self):
+        self.logger.removeHandler(self.handler)
+
+    def test_debug_logging(self):
+        log_message = "Foo message"
+        extra = {"important_info": "foo"}
+        self.spider.log_debug(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "DEBUG"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_info_logging(self):
+        log_message = "Bar message"
+        extra = {"important_info": "bar"}
+        self.spider.log_info(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "INFO"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_warning_logging(self):
+        log_message = "Baz message"
+        extra = {"important_info": "baz"}
+        self.spider.log_warning(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "WARNING"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_error_logging(self):
+        log_message = "Foo bar message"
+        extra = {"important_info": "foo bar"}
+        self.spider.log_error(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "ERROR"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_critical_logging(self):
+        log_message = "Foo bar baz message"
+        extra = {"important_info": "foo bar baz"}
+        self.spider.log_critical(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "CRITICAL"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_overwrite_spider_extra(self):
+        log_message = "Foo message"
+        extra = {"important_info": "foo", "spider": "shouldn't change"}
+        self.spider.log_error(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "ERROR"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]

tests/test_utils_project.py

@@ -6,6 +6,7 @@ import unittest
 import warnings
 from pathlib import Path
 
+from scrapy.utils.misc import set_environ
 from scrapy.utils.project import data_path, get_project_settings
 
 
@@ -38,20 +39,6 @@ class ProjectUtilsTest(unittest.TestCase):
             self.assertEqual(abspath, data_path(abspath))
 
 
-@contextlib.contextmanager
-def set_env(**update):
-    modified = set(update.keys()) & set(os.environ.keys())
-    update_after = {k: os.environ[k] for k in modified}
-    remove_after = frozenset(k for k in update if k not in os.environ)
-    try:
-        os.environ.update(update)
-        yield
-    finally:
-        os.environ.update(update_after)
-        for k in remove_after:
-            os.environ.pop(k)
-
-
 class GetProjectSettingsTestCase(unittest.TestCase):
     def test_valid_envvar(self):
         value = "tests.test_cmdline.settings"
@@ -60,7 +47,7 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         }
         with warnings.catch_warnings():
             warnings.simplefilter("error")
-            with set_env(**envvars):
+            with set_environ(**envvars):
                 settings = get_project_settings()
 
         assert settings.get("SETTINGS_MODULE") == value
@@ -69,7 +56,7 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         envvars = {
             "SCRAPY_FOO": "bar",
         }
-        with set_env(**envvars):
+        with set_environ(**envvars):
             settings = get_project_settings()
 
         assert settings.get("SCRAPY_FOO") is None
@@ -80,7 +67,7 @@ class GetProjectSettingsTestCase(unittest.TestCase):
             "SCRAPY_FOO": "bar",
             "SCRAPY_SETTINGS_MODULE": value,
         }
-        with set_env(**envvars):
+        with set_environ(**envvars):
             settings = get_project_settings()
         assert settings.get("SETTINGS_MODULE") == value
         assert settings.get("SCRAPY_FOO") is None

tests/test_utils_python.py

@@ -239,8 +239,11 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"])
 
         if platform.python_implementation() == "CPython":
-            # doesn't work on CPython: https://bugs.python.org/issue42785
+            # This didn't work on older versions of CPython: https://github.com/python/cpython/issues/86951
-            self.assertEqual(get_func_args(operator.itemgetter(2)), [])
+            self.assertIn(
+                get_func_args(operator.itemgetter(2), stripself=True),
+                [[], ["args", "kwargs"]],
+            )
         elif platform.python_implementation() == "PyPy":
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself=True), ["obj"]

tox.ini

@@ -16,6 +16,9 @@ deps =
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
     mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
+    # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by
+    # mitmproxy.
+    werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -71,6 +74,7 @@ commands =
     twine check dist/*
 
 [pinned]
+basepython = python3.8
 deps =
     cryptography==36.0.0
     cssselect==0.9.1
@@ -97,7 +101,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
 
 [testenv:pinned]
-basepython = python3.8
+basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
     PyDispatcher==2.0.5
@@ -107,7 +111,7 @@ setenv =
 commands = {[pinned]commands}
 
 [testenv:windows-pinned]
-basepython = python3
+basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
     PyDispatcher==2.0.5
@@ -130,13 +134,14 @@ deps =
     Twisted[http2]
 
 [testenv:extra-deps-pinned]
-basepython = python3.8
+basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
     boto3==1.20.0
     google-cloud-storage==1.29.0
     Pillow==7.1.0
     robotexclusionrulesparser==1.6.2
+    brotlipy
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
@@ -147,6 +152,7 @@ commands =
     {[testenv]commands} --reactor=asyncio
 
 [testenv:asyncio-pinned]
+basepython = {[pinned]basepython}
 deps = {[testenv:pinned]deps}
 commands = {[pinned]commands} --reactor=asyncio
 install_command = {[pinned]install_command}
@@ -159,12 +165,12 @@ commands =
     pytest {posargs:--durations=10 docs scrapy tests}
 
 [testenv:pypy3-pinned]
-basepython = {[testenv:pypy3]basepython}
+basepython = pypy3.8
 deps =
     {[pinned]deps}
     PyPyDispatcher==2.1.0
 commands =
-    pytest --durations=10 scrapy tests
+    pytest {posargs:--durations=10 scrapy tests}
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
@@ -212,7 +218,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
 
 [testenv:botocore-pinned]
-basepython = python3.8
+basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
     botocore==1.4.87