Merge remote-tracking branch 'origin/master' into addons

.github/workflows/tests-macos.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/tests-ubuntu.yml

@@ -8,9 +8,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.8
-          env:
-            TOXENV: py
         - python-version: 3.9
           env:
             TOXENV: py
@@ -28,19 +25,19 @@ jobs:
             TOXENV: pypy3
 
         # pinned deps
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: asyncio-pinned
-        - python-version: pypy3.7
+        - python-version: pypy3.8
           env:
             TOXENV: pypy3-pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: extra-deps-pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: botocore-pinned
 

.github/workflows/tests-windows.yml

@@ -8,12 +8,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.7
-          env:
-            TOXENV: windows-pinned
         - python-version: 3.8
           env:
-            TOXENV: py
+            TOXENV: windows-pinned
         - python-version: 3.9
           env:
             TOXENV: py

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4  # 6.0.0 drops Python 3.7 support
+  rev: 6.0.0
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
@@ -13,7 +13,7 @@ repos:
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.11.5  # 5.12 drops Python 3.7 support
+  rev: 5.12.0
   hooks:
   - id: isort
 - repo: https://github.com/adamchainz/blacken-docs

README.rst

@@ -58,7 +58,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.7+
+* Python 3.8+
 * Works on Linux, Windows, macOS, BSD
 
 Install

docs/contributing.rst

@@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
 To run the tests on a specific :doc:`tox <tox:index>` environment, use
 ``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.7 use::
+the tests with Python 3.10 use::
 
-    tox -e py37
+    tox -e py310
 
 You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
-    tox -e py37,py38 -p auto
+    tox -e py39,py310 -p auto
 
 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
     tox -- scrapy tests -x  # stop after first failure
 
 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
 
-    tox -e py37 -- scrapy tests -n auto
+    tox -e py310 -- scrapy tests -n auto
 
 To see coverage report install :doc:`coverage <coverage:index>`
 (``pip install coverage``) and run:
@@ -322,4 +322,4 @@ And their unit-tests are in::
 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
 .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
-.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
+.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy

docs/intro/install.rst

@@ -9,7 +9,7 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.7+, either the CPython implementation (default) or
+Scrapy requires Python 3.8+, either the CPython implementation (default) or
 the PyPy implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:

docs/topics/components.rst

@@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use
 
 .. code-block:: python
 
-    from pkg_resources import parse_version
+    from packaging.version import parse as parse_version
 
     import scrapy
 

docs/topics/downloader-middleware.rst

@@ -915,6 +915,7 @@ settings (see the settings documentation for more info):
 * :setting:`RETRY_ENABLED`
 * :setting:`RETRY_TIMES`
 * :setting:`RETRY_HTTP_CODES`
+* :setting:`RETRY_EXCEPTIONS`
 
 .. reqmeta:: dont_retry
 
@@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
 it is a common code used to indicate server overload. It is not included by
 default because HTTP specs say so.
 
+.. setting:: RETRY_EXCEPTIONS
+
+RETRY_EXCEPTIONS
+^^^^^^^^^^^^^^^^
+
+Default::
+
+    [
+        'twisted.internet.defer.TimeoutError',
+        'twisted.internet.error.TimeoutError',
+        'twisted.internet.error.DNSLookupError',
+        'twisted.internet.error.ConnectionRefusedError',
+        'twisted.internet.error.ConnectionDone',
+        'twisted.internet.error.ConnectError',
+        'twisted.internet.error.ConnectionLost',
+        'twisted.internet.error.TCPTimedOutError',
+        'twisted.web.client.ResponseFailed',
+        IOError,
+        'scrapy.core.downloader.handlers.http11.TunnelError',
+    ]
+
+List of exceptions to retry.
+
+Each list entry may be an exception type or its import path as a string.
+
+An exception will not be caught when the exception type is not in
+:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
+has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
+exception propagation, see
+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
+
 .. setting:: RETRY_PRIORITY_ADJUST
 
 RETRY_PRIORITY_ADJUST

docs/topics/feed-exports.rst

@@ -156,8 +156,8 @@ The feeds are stored in the local filesystem.
 -   Required external libraries: none
 
 Note that for the local filesystem storage (only) you can omit the scheme if
-you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
-systems though.
+you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
+Alternatively you can also use a :class:`pathlib.Path` object.
 
 .. _topics-feed-storage-ftp:
 

docs/topics/request-response.rst

@@ -1103,9 +1103,10 @@ Response objects
         through all :ref:`Downloader Middlewares <topics-downloader-middleware>`.
         In particular, this means that:
 
-        - HTTP redirections will cause the original request (to the URL before
-          redirection) to be assigned to the redirected response (with the final
-          URL after redirection).
+        - HTTP redirections will create a new request from the request before
+          redirection. It has the majority of the same metadata and original
+          request attributes and gets assigned to the redirected response
+          instead of the propagation of the original request.
 
         - Response.request.url doesn't always equal Response.url
 

docs/utils/linkfix.py

@@ -30,7 +30,7 @@ def main():
     try:
         with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
             output_lines = out.readlines()
-    except IOError:
+    except OSError:
         print("linkcheck output not found; please run linkcheck first.")
         sys.exit(1)
 

scrapy/__init__.py

@@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 7):
-    print(f"Scrapy {__version__} requires Python 3.7+")
+if sys.version_info < (3, 8):
+    print(f"Scrapy {__version__} requires Python 3.8+")
     sys.exit(1)
 
 

scrapy/cmdline.py

@@ -3,8 +3,7 @@ import cProfile
 import inspect
 import os
 import sys
-
-import pkg_resources
+from importlib.metadata import entry_points
 
 import scrapy
 from scrapy.addons import AddonManager
@@ -50,7 +49,7 @@ def _get_commands_from_module(module, inproject):
 
 def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
-    for entry_point in pkg_resources.iter_entry_points(group):
+    for entry_point in entry_points().get(group, {}):
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()

scrapy/crawler.py

@@ -4,7 +4,7 @@ import logging
 import pprint
 import signal
 import warnings
-from typing import TYPE_CHECKING, Optional, Set, Type, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union
 
 from twisted.internet import defer
 from zope.interface.exceptions import DoesNotImplement
@@ -26,6 +26,7 @@ from scrapy.interfaces import ISpiderLoader
 from scrapy.logformatter import LogFormatter
 from scrapy.settings import Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
+from scrapy.spiderloader import SpiderLoader
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.log import (
     LogCounterHandler,
@@ -180,7 +181,7 @@ class CrawlerRunner:
     )
 
     @staticmethod
-    def _get_spider_loader(settings):
+    def _get_spider_loader(settings) -> SpiderLoader:
         """Get SpiderLoader instance from settings"""
         cls_path = settings.get("SPIDER_LOADER_CLASS")
         loader_cls = load_object(cls_path)
@@ -199,7 +200,11 @@ class CrawlerRunner:
             )
         return loader_cls.from_settings(settings.frozencopy())
 
-    def __init__(self, settings=None, addons: Optional[AddonManager] = None):
+    def __init__(
+        self,
+        settings: Union[Dict[str, Any], Settings, None] = None,
+        addons: Optional[AddonManager] = None,
+    ):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         self.settings = settings
@@ -262,7 +267,9 @@ class CrawlerRunner:
 
         return d.addBoth(_done)
 
-    def create_crawler(self, crawler_or_spidercls):
+    def create_crawler(
+        self, crawler_or_spidercls: Union[Type[Spider], str, Crawler]
+    ) -> Crawler:
         """
         Return a :class:`~scrapy.crawler.Crawler` object.
 
@@ -282,7 +289,7 @@ class CrawlerRunner:
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)
 
-    def _create_crawler(self, spidercls):
+    def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler:
         if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings, addons=self.addons)

scrapy/downloadermiddlewares/decompression.py

@@ -63,7 +63,7 @@ class DecompressionMiddleware:
         archive = BytesIO(response.body)
         try:
             body = gzip.GzipFile(fileobj=archive).read()
-        except IOError:
+        except OSError:
             return
 
         respcls = responsetypes.from_args(body=body)
@@ -72,7 +72,7 @@ class DecompressionMiddleware:
     def _is_bzip2(self, response):
         try:
             body = bz2.decompress(response.body)
-        except IOError:
+        except OSError:
             return
 
         respcls = responsetypes.from_args(body=body)

scrapy/downloadermiddlewares/httpcache.py

@@ -37,7 +37,7 @@ class HttpCacheMiddleware:
         ConnectionLost,
         TCPTimedOutError,
         ResponseFailed,
-        IOError,
+        OSError,
     )
 
     def __init__(self, settings: Settings, stats: StatsCollector) -> None:

scrapy/downloadermiddlewares/retry.py

@@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+import warnings
 from logging import Logger, getLogger
 from typing import Optional, Union
 
-from twisted.internet import defer
-from twisted.internet.error import (
-    ConnectError,
-    ConnectionDone,
-    ConnectionLost,
-    ConnectionRefusedError,
-    DNSLookupError,
-    TCPTimedOutError,
-    TimeoutError,
-)
-from twisted.web.client import ResponseFailed
-
-from scrapy.core.downloader.handlers.http11 import TunnelError
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http.request import Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
 
 retry_logger = getLogger(__name__)
 
 
+class BackwardsCompatibilityMetaclass(type):
+    @property
+    def EXCEPTIONS_TO_RETRY(cls):
+        warnings.warn(
+            "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
+            "Use the RETRY_EXCEPTIONS setting instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return tuple(
+            load_object(x) if isinstance(x, str) else x
+            for x in Settings().getlist("RETRY_EXCEPTIONS")
+        )
+
+
 def get_retry_request(
     request: Request,
     *,
@@ -121,23 +126,7 @@ def get_retry_request(
     return None
 
 
-class RetryMiddleware:
-    # IOError is raised by the HttpCompression middleware when trying to
-    # decompress an empty response
-    EXCEPTIONS_TO_RETRY = (
-        defer.TimeoutError,
-        TimeoutError,
-        DNSLookupError,
-        ConnectionRefusedError,
-        ConnectionDone,
-        ConnectError,
-        ConnectionLost,
-        TCPTimedOutError,
-        ResponseFailed,
-        IOError,
-        TunnelError,
-    )
-
+class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
     def __init__(self, settings):
         if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
@@ -147,6 +136,16 @@ class RetryMiddleware:
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
+        if not hasattr(
+            self, "EXCEPTIONS_TO_RETRY"
+        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+            self.exceptions_to_retry = tuple(
+                load_object(x) if isinstance(x, str) else x
+                for x in settings.getlist("RETRY_EXCEPTIONS")
+            )
+        else:
+            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
@@ -160,7 +159,7 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
+        if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
             "dont_retry", False
         ):
             return self._retry(request, exception, spider)

scrapy/extensions/feedexport.py

@@ -382,7 +382,9 @@ class FeedExporter:
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-            uri = str(self.settings["FEED_URI"])  # handle pathlib.Path objects
+            uri = self.settings["FEED_URI"]
+            # handle pathlib.Path objects
+            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
             feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
@@ -392,7 +394,8 @@ class FeedExporter:
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
         for uri, feed_options in self.settings.getdict("FEEDS").items():
-            uri = str(uri)  # handle pathlib.Path objects
+            # handle pathlib.Path objects
+            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
             )

scrapy/http/headers.py

@@ -2,7 +2,7 @@ from collections.abc import Mapping
 
 from w3lib.http import headers_dict_to_raw
 
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
 from scrapy.utils.python import to_unicode
 
 
@@ -88,7 +88,7 @@ class Headers(CaselessDict):
         """Return headers as a CaselessDict with unicode keys
         and unicode values. Multiple values are joined with ','.
         """
-        return CaselessDict(
+        return CaseInsensitiveDict(
             (
                 to_unicode(key, encoding=self.encoding),
                 to_unicode(b",".join(value), encoding=self.encoding),

scrapy/pipelines/files.py

@@ -28,7 +28,7 @@ from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import md5sum
@@ -155,7 +155,7 @@ class S3FilesStore:
     def _headers_to_botocore_kwargs(self, headers):
         """Convert headers to botocore keyword arguments."""
         # This is required while we need to support both boto and botocore.
-        mapping = CaselessDict(
+        mapping = CaseInsensitiveDict(
             {
                 "Content-Type": "ContentType",
                 "Cache-Control": "CacheControl",

scrapy/resolver.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
 from twisted.internet.interfaces import (
@@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider
 from scrapy.utils.datatypes import LocalCache
 
 # TODO: cache misses
-dnscache = LocalCache(10000)
+dnscache: LocalCache[str, Any] = LocalCache(10000)
 
 
 @implementer(IResolverSimple)
@@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver):
     def install_on_reactor(self):
         self.reactor.installResolver(self)
 
-    def getHostByName(self, name, timeout=None):
+    def getHostByName(self, name: str, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
         # in Twisted<=16.6, getHostByName() is always called with
@@ -110,7 +112,7 @@ class CachingHostnameResolver:
     def resolveHostName(
         self,
         resolutionReceiver,
-        hostName,
+        hostName: str,
         portNumber=0,
         addressTypes=None,
         transportSemantics="TCP",

scrapy/settings/__init__.py

@@ -76,6 +76,8 @@ class BaseSettings(MutableMapping):
     highest priority will be retrieved.
     """
 
+    __default = object()
+
     def __init__(self, values=None, priority="project"):
         self.frozen = False
         self.attributes = {}
@@ -446,6 +448,18 @@ class BaseSettings(MutableMapping):
         else:
             p.text(pformat(self.copy_to_dict()))
 
+    def pop(self, name, default=__default):
+        try:
+            value = self.attributes[name]
+        except KeyError:
+            if default is self.__default:
+                raise
+
+            return SettingsAttribute(default, get_settings_priority("project"))
+        else:
+            self.__delitem__(name)
+            return value
+
 
 class Settings(BaseSettings):
     """

scrapy/settings/default_settings.py

@@ -260,6 +260,21 @@ RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
+RETRY_EXCEPTIONS = [
+    "twisted.internet.defer.TimeoutError",
+    "twisted.internet.error.TimeoutError",
+    "twisted.internet.error.DNSLookupError",
+    "twisted.internet.error.ConnectionRefusedError",
+    "twisted.internet.error.ConnectionDone",
+    "twisted.internet.error.ConnectError",
+    "twisted.internet.error.ConnectionLost",
+    "twisted.internet.error.TCPTimedOutError",
+    "twisted.web.client.ResponseFailed",
+    # OSError is raised by the HttpCompression middleware when trying to
+    # decompress an empty response
+    OSError,
+    "scrapy.core.downloader.handlers.http11.TunnelError",
+]
 
 ROBOTSTXT_OBEY = False
 ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"

scrapy/spiderloader.py

@@ -1,10 +1,13 @@
 import traceback
 import warnings
 from collections import defaultdict
+from typing import DefaultDict, Dict, List, Tuple, Type
 
 from zope.interface import implementer
 
+from scrapy import Spider
 from scrapy.interfaces import ISpiderLoader
+from scrapy.settings import BaseSettings
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.spider import iter_spider_classes
 
@@ -16,11 +19,11 @@ class SpiderLoader:
     in a Scrapy project.
     """
 
-    def __init__(self, settings):
+    def __init__(self, settings: BaseSettings):
         self.spider_modules = settings.getlist("SPIDER_MODULES")
         self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY")
-        self._spiders = {}
-        self._found = defaultdict(list)
+        self._spiders: Dict[str, Type[Spider]] = {}
+        self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list)
         self._load_all_spiders()
 
     def _check_name_duplicates(self):
@@ -68,7 +71,7 @@ class SpiderLoader:
     def from_settings(cls, settings):
         return cls(settings)
 
-    def load(self, spider_name):
+    def load(self, spider_name: str) -> Type[Spider]:
         """
         Return the Spider class for the given spider name. If the spider
         name is not found, raise a KeyError.

scrapy/utils/boto.py

@@ -1,7 +1,7 @@
 """Boto/botocore helpers"""
 
 
-def is_botocore_available():
+def is_botocore_available() -> bool:
     try:
         import botocore  # noqa: F401
 

scrapy/utils/datatypes.py

@@ -6,13 +6,32 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import collections
+import warnings
 import weakref
 from collections.abc import Mapping
+from typing import Any, AnyStr, Optional, OrderedDict, Sequence, TypeVar
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
 
 
 class CaselessDict(dict):
     __slots__ = ()
 
+    def __new__(cls, *args, **kwargs):
+        from scrapy.http.headers import Headers
+
+        if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
+            warnings.warn(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        return super().__new__(cls, *args, **kwargs)
+
     def __init__(self, seq=None):
         super().__init__()
         if seq:
@@ -64,17 +83,59 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class LocalCache(collections.OrderedDict):
+class CaseInsensitiveDict(collections.UserDict):
+    """A dict-like structure that accepts strings or bytes
+    as keys and allows case-insensitive lookups.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        self._keys: dict = {}
+        super().__init__(*args, **kwargs)
+
+    def __getitem__(self, key: AnyStr) -> Any:
+        normalized_key = self._normkey(key)
+        return super().__getitem__(self._keys[normalized_key.lower()])
+
+    def __setitem__(self, key: AnyStr, value: Any) -> None:
+        normalized_key = self._normkey(key)
+        try:
+            lower_key = self._keys[normalized_key.lower()]
+            del self[lower_key]
+        except KeyError:
+            pass
+        super().__setitem__(normalized_key, self._normvalue(value))
+        self._keys[normalized_key.lower()] = normalized_key
+
+    def __delitem__(self, key: AnyStr) -> None:
+        normalized_key = self._normkey(key)
+        stored_key = self._keys.pop(normalized_key.lower())
+        super().__delitem__(stored_key)
+
+    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+        normalized_key = self._normkey(key)
+        return normalized_key.lower() in self._keys
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: {super().__repr__()}>"
+
+    def _normkey(self, key: AnyStr) -> AnyStr:
+        return key
+
+    def _normvalue(self, value: Any) -> Any:
+        return value
+
+
+class LocalCache(OrderedDict[_KT, _VT]):
     """Dictionary with a finite number of keys.
 
     Older items expires first.
     """
 
-    def __init__(self, limit=None):
+    def __init__(self, limit: Optional[int] = None):
         super().__init__()
-        self.limit = limit
+        self.limit: Optional[int] = limit
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: _KT, value: _VT) -> None:
         if self.limit:
             while len(self) >= self.limit:
                 self.popitem(last=False)
@@ -93,17 +154,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     it cannot be instantiated with an initial dictionary.
     """
 
-    def __init__(self, limit=None):
+    def __init__(self, limit: Optional[int] = None):
         super().__init__()
-        self.data = LocalCache(limit=limit)
+        self.data: LocalCache = LocalCache(limit=limit)
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: _KT, value: _VT) -> None:
         try:
             super().__setitem__(key, value)
         except TypeError:
             pass  # key is not weak-referenceable, skip caching
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: _KT) -> Optional[_VT]:  # type: ignore[override]
         try:
             return super().__getitem__(key)
         except (TypeError, KeyError):
@@ -113,8 +174,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 class SequenceExclude:
     """Object to test if an item is NOT within some sequence."""
 
-    def __init__(self, seq):
-        self.seq = seq
+    def __init__(self, seq: Sequence):
+        self.seq: Sequence = seq
 
-    def __contains__(self, item):
+    def __contains__(self, item: Any) -> bool:
         return item not in self.seq

scrapy/utils/decorators.py

@@ -1,19 +1,21 @@
 import warnings
 from functools import wraps
+from typing import Any, Callable
 
 from twisted.internet import defer, threads
+from twisted.internet.defer import Deferred
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def deprecated(use_instead=None):
+def deprecated(use_instead: Any = None) -> Callable:
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used."""
 
-    def deco(func):
+    def deco(func: Callable) -> Callable:
         @wraps(func)
-        def wrapped(*args, **kwargs):
+        def wrapped(*args: Any, **kwargs: Any) -> Any:
             message = f"Call to deprecated function {func.__name__}."
             if use_instead:
                 message += f" Use {use_instead} instead."
@@ -28,23 +30,23 @@ def deprecated(use_instead=None):
     return deco
 
 
-def defers(func):
+def defers(func: Callable) -> Callable[..., Deferred]:
     """Decorator to make sure a function always returns a deferred"""
 
     @wraps(func)
-    def wrapped(*a, **kw):
+    def wrapped(*a: Any, **kw: Any) -> Deferred:
         return defer.maybeDeferred(func, *a, **kw)
 
     return wrapped
 
 
-def inthread(func):
+def inthread(func: Callable) -> Callable[..., Deferred]:
     """Decorator to call a function in a thread and return a deferred with the
     result
     """
 
     @wraps(func)
-    def wrapped(*a, **kw):
+    def wrapped(*a: Any, **kw: Any) -> Deferred:
         return threads.deferToThread(func, *a, **kw)
 
     return wrapped

scrapy/utils/defer.py

@@ -9,13 +9,16 @@ from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
+    AsyncIterator,
     Callable,
     Coroutine,
+    Dict,
     Generator,
     Iterable,
     Iterator,
     List,
     Optional,
+    Tuple,
     Union,
     cast,
 )
@@ -44,7 +47,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     return d
 
 
-def defer_succeed(result) -> Deferred:
+def defer_succeed(result: Any) -> Deferred:
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
 
@@ -58,7 +61,7 @@ def defer_succeed(result) -> Deferred:
     return d
 
 
-def defer_result(result) -> Deferred:
+def defer_result(result: Any) -> Deferred:
     if isinstance(result, Deferred):
         return result
     if isinstance(result, failure.Failure):
@@ -66,7 +69,7 @@ def defer_result(result) -> Deferred:
     return defer_succeed(result)
 
 
-def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
+def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred:
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
     """
@@ -84,7 +87,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
 
 
 def parallel(
-    iterable: Iterable, count: int, callable: Callable, *args, **named
+    iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any
 ) -> Deferred:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
@@ -146,14 +149,14 @@ class _AsyncCooperatorAdapter(Iterator):
         self,
         aiterable: AsyncIterable,
         callable: Callable,
-        *callable_args,
-        **callable_kwargs
+        *callable_args: Any,
+        **callable_kwargs: Any,
     ):
-        self.aiterator = aiterable.__aiter__()
-        self.callable = callable
-        self.callable_args = callable_args
-        self.callable_kwargs = callable_kwargs
-        self.finished = False
+        self.aiterator: AsyncIterator = aiterable.__aiter__()
+        self.callable: Callable = callable
+        self.callable_args: Tuple[Any, ...] = callable_args
+        self.callable_kwargs: Dict[str, Any] = callable_kwargs
+        self.finished: bool = False
         self.waiting_deferreds: List[Deferred] = []
         self.anext_deferred: Optional[Deferred] = None
 
@@ -201,7 +204,11 @@ class _AsyncCooperatorAdapter(Iterator):
 
 
 def parallel_async(
-    async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named
+    async_iterable: AsyncIterable,
+    count: int,
+    callable: Callable,
+    *args: Any,
+    **named: Any,
 ) -> Deferred:
     """Like parallel but for async iterators"""
     coop = Cooperator()
@@ -210,7 +217,9 @@ def parallel_async(
     return dl
 
 
-def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
+def process_chain(
+    callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any
+) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
     d: Deferred = Deferred()
     for x in callbacks:
@@ -220,7 +229,11 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
 
 
 def process_chain_both(
-    callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
+    callbacks: Iterable[Callable],
+    errbacks: Iterable[Callable],
+    input: Any,
+    *a: Any,
+    **kw: Any,
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d: Deferred = Deferred()
@@ -240,7 +253,9 @@ def process_chain_both(
     return d
 
 
-def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
+def process_parallel(
+    callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any
+) -> Deferred:
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
@@ -250,7 +265,9 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred
     return d
 
 
-def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
+def iter_errback(
+    iterable: Iterable, errback: Callable, *a: Any, **kw: Any
+) -> Generator:
     """Wraps an iterable calling an errback if an error is caught while
     iterating it.
     """
@@ -265,7 +282,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
 
 
 async def aiter_errback(
-    aiterable: AsyncIterable, errback: Callable, *a, **kw
+    aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any
 ) -> AsyncGenerator:
     """Wraps an async iterable calling an errback if an error is caught while
     iterating it. Similar to scrapy.utils.defer.iter_errback()
@@ -280,7 +297,7 @@ async def aiter_errback(
             errback(failure.Failure(), *a, **kw)
 
 
-def deferred_from_coro(o) -> Any:
+def deferred_from_coro(o: Any) -> Any:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, Deferred):
         return o
@@ -303,13 +320,13 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
     """
 
     @wraps(coro_f)
-    def f(*coro_args, **coro_kwargs):
+    def f(*coro_args: Any, **coro_kwargs: Any) -> Any:
         return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
 
     return f
 
 
-def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
+def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred:
     """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)

scrapy/utils/deprecate.py

@@ -2,12 +2,12 @@
 
 import inspect
 import warnings
-from typing import List, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Type, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def attribute(obj, oldattr, newattr, version="0.12"):
+def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None:
     cname = obj.__class__.__name__
     warnings.warn(
         f"{cname}.{oldattr} attribute is deprecated and will be no longer supported "
@@ -18,16 +18,16 @@ def attribute(obj, oldattr, newattr, version="0.12"):
 
 
 def create_deprecated_class(
-    name,
-    new_class,
-    clsdict=None,
-    warn_category=ScrapyDeprecationWarning,
-    warn_once=True,
-    old_class_path=None,
-    new_class_path=None,
-    subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
-    instance_warn_message="{cls} is deprecated, instantiate {new} instead.",
-):
+    name: str,
+    new_class: type,
+    clsdict: Optional[Dict[str, Any]] = None,
+    warn_category: Type[Warning] = ScrapyDeprecationWarning,
+    warn_once: bool = True,
+    old_class_path: Optional[str] = None,
+    new_class_path: Optional[str] = None,
+    subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.",
+    instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.",
+) -> type:
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
     Subclasses of ``new_class`` are considered subclasses of this class.
@@ -53,17 +53,20 @@ def create_deprecated_class(
     OldName.
     """
 
-    class DeprecatedClass(new_class.__class__):
-        deprecated_class = None
-        warned_on_subclass = False
+    # https://github.com/python/mypy/issues/4177
+    class DeprecatedClass(new_class.__class__):  # type: ignore[misc, name-defined]
+        deprecated_class: Optional[type] = None
+        warned_on_subclass: bool = False
 
-        def __new__(metacls, name, bases, clsdict_):
+        def __new__(
+            metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]
+        ) -> type:
             cls = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
 
-        def __init__(cls, name, bases, clsdict_):
+        def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]):
             meta = cls.__class__
             old = meta.deprecated_class
             if old in bases and not (warn_once and meta.warned_on_subclass):
@@ -81,10 +84,10 @@ def create_deprecated_class(
         # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
         # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
         # for implementation details
-        def __instancecheck__(cls, inst):
+        def __instancecheck__(cls, inst: Any) -> bool:
             return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__))
 
-        def __subclasscheck__(cls, sub):
+        def __subclasscheck__(cls, sub: type) -> bool:
             if cls is not DeprecatedClass.deprecated_class:
                 # we should do the magic only if second `issubclass` argument
                 # is the deprecated class itself - subclasses of the
@@ -98,7 +101,7 @@ def create_deprecated_class(
             mro = getattr(sub, "__mro__", ())
             return any(c in {cls, new_class} for c in mro)
 
-        def __call__(cls, *args, **kwargs):
+        def __call__(cls, *args: Any, **kwargs: Any) -> Any:
             old = DeprecatedClass.deprecated_class
             if cls is old:
                 msg = instance_warn_message.format(
@@ -125,7 +128,7 @@ def create_deprecated_class(
     return deprecated_cls
 
 
-def _clspath(cls, forced=None):
+def _clspath(cls: type, forced: Optional[str] = None) -> str:
     if forced is not None:
         return forced
     return f"{cls.__module__}.{cls.__name__}"
@@ -134,7 +137,17 @@ def _clspath(cls, forced=None):
 DEPRECATION_RULES: List[Tuple[str, str]] = []
 
 
-def update_classpath(path):
+@overload
+def update_classpath(path: str) -> str:
+    ...
+
+
+@overload
+def update_classpath(path: Any) -> Any:
+    ...
+
+
+def update_classpath(path: Any) -> Any:
     """Update a deprecated path from an object with its new location"""
     for prefix, replacement in DEPRECATION_RULES:
         if isinstance(path, str) and path.startswith(prefix):
@@ -147,7 +160,7 @@ def update_classpath(path):
     return path
 
 
-def method_is_overridden(subclass, base_class, method_name):
+def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool:
     """
     Return True if a method named ``method_name`` of a ``base_class``
     is overridden in a ``subclass``.

scrapy/utils/display.py

@@ -6,17 +6,18 @@ import ctypes
 import platform
 import sys
 from pprint import pformat as pformat_
+from typing import Any
 
 from packaging.version import Version as parse_version
 
 
-def _enable_windows_terminal_processing():
+def _enable_windows_terminal_processing() -> bool:
     # https://stackoverflow.com/a/36760881
-    kernel32 = ctypes.windll.kernel32
+    kernel32 = ctypes.windll.kernel32  # type: ignore[attr-defined]
     return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
 
 
-def _tty_supports_color():
+def _tty_supports_color() -> bool:
     if sys.platform != "win32":
         return True
 
@@ -28,7 +29,7 @@ def _tty_supports_color():
     return _enable_windows_terminal_processing()
 
 
-def _colorize(text, colorize=True):
+def _colorize(text: str, colorize: bool = True) -> str:
     if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
@@ -42,9 +43,9 @@ def _colorize(text, colorize=True):
         return highlight(text, PythonLexer(), TerminalFormatter())
 
 
-def pformat(obj, *args, **kwargs):
+def pformat(obj: Any, *args: Any, **kwargs: Any) -> str:
     return _colorize(pformat_(obj), kwargs.pop("colorize", True))
 
 
-def pprint(obj, *args, **kwargs):
+def pprint(obj: Any, *args: Any, **kwargs: Any) -> None:
     print(pformat(obj, *args, **kwargs))

scrapy/utils/engine.py

@@ -2,9 +2,13 @@
 
 # used in global tests code
 from time import time  # noqa: F401
+from typing import TYPE_CHECKING, Any, List, Tuple
+
+if TYPE_CHECKING:
+    from scrapy.core.engine import ExecutionEngine
 
 
-def get_engine_status(engine):
+def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
     """Return a report of the current engine status"""
     tests = [
         "time()-engine.start_time",
@@ -23,7 +27,7 @@ def get_engine_status(engine):
         "engine.scraper.slot.needs_backout()",
     ]
 
-    checks = []
+    checks: List[Tuple[str, Any]] = []
     for test in tests:
         try:
             checks += [(test, eval(test))]
@@ -33,7 +37,7 @@ def get_engine_status(engine):
     return checks
 
 
-def format_engine_status(engine=None):
+def format_engine_status(engine: "ExecutionEngine") -> str:
     checks = get_engine_status(engine)
     s = "Execution engine status\n\n"
     for test, result in checks:
@@ -43,5 +47,5 @@ def format_engine_status(engine=None):
     return s
 
 
-def print_engine_status(engine):
+def print_engine_status(engine: "ExecutionEngine") -> None:
     print(format_engine_status(engine))

scrapy/utils/gz.py

@@ -1,34 +1,32 @@
 import struct
 from gzip import GzipFile
 from io import BytesIO
+from typing import List
+
+from scrapy.http import Response
 
 
-def gunzip(data):
+def gunzip(data: bytes) -> bytes:
     """Gunzip the given data and return as much data as possible.
 
     This is resilient to CRC checksum errors.
     """
     f = GzipFile(fileobj=BytesIO(data))
-    output_list = []
+    output_list: List[bytes] = []
     chunk = b"."
     while chunk:
         try:
             chunk = f.read1(8196)
             output_list.append(chunk)
-        except (IOError, EOFError, struct.error):
+        except (OSError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
-            # some pages are quite small so output_list is empty and f.extrabuf
-            # contains the whole page content
-            if output_list or getattr(f, "extrabuf", None):
-                try:
-                    output_list.append(f.extrabuf[-f.extrasize :])
-                finally:
-                    break
-            else:
-                raise
+            # some pages are quite small so output_list is empty
+            if output_list:
+                break
+            raise
     return b"".join(output_list)
 
 
-def gzip_magic_number(response):
+def gzip_magic_number(response: Response) -> bool:
     return response.body[:3] == b"\x1f\x8b\x08"

scrapy/utils/misc.py

@@ -10,7 +10,21 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from typing import TYPE_CHECKING, Any, Callable, Union
+from types import ModuleType
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Deque,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Pattern,
+    Union,
+    cast,
+)
 
 from w3lib.html import replace_entities
 
@@ -26,7 +40,7 @@ if TYPE_CHECKING:
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 
 
-def arg_to_iter(arg):
+def arg_to_iter(arg: Any) -> Iterable[Any]:
     """Convert an argument to an iterable. The argument can be a None, single
     value, or an iterable.
 
@@ -35,7 +49,7 @@ def arg_to_iter(arg):
     if arg is None:
         return []
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return arg
+        return cast(Iterable[Any], arg)
     return [arg]
 
 
@@ -88,7 +102,7 @@ def load_module_or_object(path):
     raise NameError(f"Could not load '{path}'")
 
 
-def walk_modules(path):
+def walk_modules(path: str) -> List[ModuleType]:
     """Loads a module and all its submodules from the given module path and
     returns them. If *any* module throws an exception while importing, that
     exception is thrown back.
@@ -96,7 +110,7 @@ def walk_modules(path):
     For example: walk_modules('scrapy.utils')
     """
 
-    mods = []
+    mods: List[ModuleType] = []
     mod = import_module(path)
     mods.append(mod)
     if hasattr(mod, "__path__"):
@@ -110,7 +124,9 @@ def walk_modules(path):
     return mods
 
 
-def extract_regex(regex, text, encoding="utf-8"):
+def extract_regex(
+    regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
+) -> List[str]:
     """Extract a list of unicode strings from the given text/encoding using the following policies:
 
     * if the regex contains a named group called "extract" that will be returned
@@ -127,9 +143,11 @@ def extract_regex(regex, text, encoding="utf-8"):
         regex = re.compile(regex, re.UNICODE)
 
     try:
-        strings = [regex.search(text).group("extract")]  # named group
+        # named group
+        strings = [regex.search(text).group("extract")]  # type: ignore[union-attr]
     except Exception:
-        strings = regex.findall(text)  # full regex or numbered groups
+        # full regex or numbered groups
+        strings = regex.findall(text)
     strings = flatten(strings)
 
     if isinstance(text, str):
@@ -139,7 +157,7 @@ def extract_regex(regex, text, encoding="utf-8"):
     ]
 
 
-def md5sum(file):
+def md5sum(file: IO) -> str:
     """Calculate the md5 checksum of a file-like object without reading its
     whole content in memory.
 
@@ -156,7 +174,7 @@ def md5sum(file):
     return m.hexdigest()
 
 
-def rel_has_nofollow(rel):
+def rel_has_nofollow(rel: Optional[str]) -> bool:
     """Return True if link rel attribute has nofollow type"""
     return rel is not None and "nofollow" in rel.replace(",", " ").split()
 
@@ -197,7 +215,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
 
 
 @contextmanager
-def set_environ(**kwargs):
+def set_environ(**kwargs: str) -> Generator[None, Any, None]:
     """Temporarily set environment variables inside the context manager and
     fully restore previous environment afterwards
     """
@@ -214,11 +232,11 @@ def set_environ(**kwargs):
                 os.environ[k] = v
 
 
-def walk_callable(node):
+def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
     """Similar to ``ast.walk``, but walks only function body and skips nested
     functions defined within the node.
     """
-    todo = deque([node])
+    todo: Deque[ast.AST] = deque([node])
     walked_func_def = False
     while todo:
         node = todo.popleft()
@@ -233,15 +251,15 @@ def walk_callable(node):
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
-def is_generator_with_return_value(callable):
+def is_generator_with_return_value(callable: Callable) -> bool:
     """
     Returns True if a callable is a generator function which includes a
     'return' statement with a value different than None, False otherwise
     """
     if callable in _generator_callbacks_cache:
-        return _generator_callbacks_cache[callable]
+        return bool(_generator_callbacks_cache[callable])
 
-    def returns_none(return_node):
+    def returns_none(return_node: ast.Return) -> bool:
         value = return_node.value
         return (
             value is None or isinstance(value, ast.NameConstant) and value.value is None
@@ -264,10 +282,10 @@ def is_generator_with_return_value(callable):
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
                 _generator_callbacks_cache[callable] = True
-                return _generator_callbacks_cache[callable]
+                return bool(_generator_callbacks_cache[callable])
 
     _generator_callbacks_cache[callable] = False
-    return _generator_callbacks_cache[callable]
+    return bool(_generator_callbacks_cache[callable])
 
 
 def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:

scrapy/utils/python.py

@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+import collections.abc
 import gc
 import inspect
 import re
@@ -12,9 +13,17 @@ from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
+    AsyncIterator,
+    Callable,
+    Dict,
+    Generator,
     Iterable,
+    Iterator,
+    List,
     Mapping,
     Optional,
+    Pattern,
+    Tuple,
     Union,
     overload,
 )
@@ -22,7 +31,7 @@ from typing import (
 from scrapy.utils.asyncgen import as_async_generator
 
 
-def flatten(x):
+def flatten(x: Iterable) -> list:
     """flatten(sequence) -> list
 
     Returns a single, flat list which contains all elements retrieved
@@ -42,7 +51,7 @@ def flatten(x):
     return list(iflatten(x))
 
 
-def iflatten(x):
+def iflatten(x: Iterable) -> Iterable:
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""
@@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool:
     return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
-def unique(list_, key=lambda x: x):
+def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list:
     """efficient function to uniquify a list preserving item order"""
     seen = set()
     result = []
@@ -124,7 +133,9 @@ def to_bytes(
     return text.encode(encoding, errors)
 
 
-def re_rsearch(pattern, text, chunk_size=1024):
+def re_rsearch(
+    pattern: Union[str, Pattern], text: str, chunk_size: int = 1024
+) -> Optional[Tuple[int, int]]:
     """
     This function does a reverse search in a text using a regular expression
     given in the attribute 'pattern'.
@@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter():
+    def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
         offset = len(text)
         while True:
             offset -= chunk_size * 1024
@@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024):
     return None
 
 
-def memoizemethod_noargs(method):
+def memoizemethod_noargs(method: Callable) -> Callable:
     """Decorator to cache the result of a method (without arguments) using a
     weak reference to its object
     """
-    cache = weakref.WeakKeyDictionary()
+    cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary()
 
     @wraps(method)
-    def new_method(self, *args, **kwargs):
+    def new_method(self: Any, *args: Any, **kwargs: Any) -> Any:
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
         return cache[self]
@@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool:
     return all(c not in _BINARYCHARS for c in data)
 
 
-def get_func_args(func, stripself=False):
+def get_func_args(func: Callable, stripself: bool = False) -> List[str]:
     """Return the argument name list of a callable object"""
     if not callable(func):
         raise TypeError(f"func must be callable, got '{type(func).__name__}'")
 
-    args = []
+    args: List[str] = []
     try:
         sig = inspect.signature(func)
     except ValueError:
@@ -217,7 +228,7 @@ def get_func_args(func, stripself=False):
     return args
 
 
-def get_spec(func):
+def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
     """Returns (args, kwargs) tuple for a function
     >>> import re
     >>> get_spec(re.match)
@@ -246,7 +257,7 @@ def get_spec(func):
     else:
         raise TypeError(f"{type(func)} is not callable")
 
-    defaults = spec.defaults or []
+    defaults: Tuple[Any, ...] = spec.defaults or ()
 
     firstdefault = len(spec.args) - len(defaults)
     args = spec.args[:firstdefault]
@@ -254,7 +265,9 @@ def get_spec(func):
     return args, kwargs
 
 
-def equal_attributes(obj1, obj2, attributes):
+def equal_attributes(
+    obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]
+) -> bool:
     """Compare two objects attributes"""
     # not attributes given return False by default
     if not attributes:
@@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable:
     ...
 
 
-def without_none_values(iterable):
+def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:
     """Return a copy of ``iterable`` with all ``None`` entries removed.
 
     If ``iterable`` is a mapping, return a dictionary where all pairs that have
     value ``None`` have been removed.
     """
-    try:
+    if isinstance(iterable, collections.abc.Mapping):
         return {k: v for k, v in iterable.items() if v is not None}
-    except AttributeError:
-        return type(iterable)((v for v in iterable if v is not None))
+    else:
+        # the iterable __init__ must take another iterable
+        return type(iterable)(v for v in iterable if v is not None)  # type: ignore[call-arg]
 
 
-def global_object_name(obj):
+def global_object_name(obj: Any) -> str:
     """
     Return full name of a global object.
 
@@ -307,14 +321,14 @@ def global_object_name(obj):
 
 if hasattr(sys, "pypy_version_info"):
 
-    def garbage_collect():
+    def garbage_collect() -> None:
         # Collecting weakreferences can take two collections on PyPy.
         gc.collect()
         gc.collect()
 
 else:
 
-    def garbage_collect():
+    def garbage_collect() -> None:
         gc.collect()
 
 
@@ -329,10 +343,10 @@ class MutableChain(Iterable):
     def extend(self, *iterables: Iterable) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator:
         return self
 
-    def __next__(self):
+    def __next__(self) -> Any:
         return next(self.data)
 
 
@@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable):
     def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
         self.data = _async_chain(self.data, _async_chain(*iterables))
 
-    def __aiter__(self):
+    def __aiter__(self) -> AsyncIterator:
         return self
 
-    async def __anext__(self):
+    async def __anext__(self) -> Any:
         return await self.data.__anext__()

scrapy/utils/reactor.py

@@ -1,7 +1,8 @@
 import asyncio
 import sys
+from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
 from contextlib import suppress
-from typing import Any, Callable, Dict, Optional, Sequence
+from typing import Any, Callable, Dict, Optional, Sequence, Type
 from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
@@ -57,7 +58,7 @@ class CallLaterOnce:
         return self._func(*self._a, **self._kw)
 
 
-def set_asyncio_event_loop_policy():
+def set_asyncio_event_loop_policy() -> None:
     """The policy functions from asyncio often behave unexpectedly,
     so we restrict their use to the absolutely essential case.
     This should only be used to install the reactor.
@@ -65,7 +66,7 @@ def set_asyncio_event_loop_policy():
     _get_asyncio_event_loop_policy()
 
 
-def get_asyncio_event_loop_policy():
+def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
     warn(
         "Call to deprecated function "
         "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
@@ -81,7 +82,7 @@ def get_asyncio_event_loop_policy():
     return _get_asyncio_event_loop_policy()
 
 
-def _get_asyncio_event_loop_policy():
+def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
     policy = asyncio.get_event_loop_policy()
     if (
         sys.version_info >= (3, 8)
@@ -93,7 +94,7 @@ def _get_asyncio_event_loop_policy():
     return policy
 
 
-def install_reactor(reactor_path, event_loop_path=None):
+def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None:
     """Installs the :mod:`~twisted.internet.reactor` with the specified
     import path. Also installs the asyncio event loop with the specified import
     path if the asyncio reactor is enabled"""
@@ -111,14 +112,14 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
-def _get_asyncio_event_loop():
+def _get_asyncio_event_loop() -> AbstractEventLoop:
     return set_asyncio_event_loop(None)
 
 
-def set_asyncio_event_loop(event_loop_path):
+def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop:
     """Sets and returns the event loop with specified import path."""
     if event_loop_path is not None:
-        event_loop_class = load_object(event_loop_path)
+        event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path)
         event_loop = event_loop_class()
         asyncio.set_event_loop(event_loop)
     else:
@@ -146,7 +147,7 @@ def set_asyncio_event_loop(event_loop_path):
     return event_loop
 
 
-def verify_installed_reactor(reactor_path):
+def verify_installed_reactor(reactor_path: str) -> None:
     """Raises :exc:`Exception` if the installed
     :mod:`~twisted.internet.reactor` does not match the specified import
     path."""
@@ -162,7 +163,7 @@ def verify_installed_reactor(reactor_path):
         raise Exception(msg)
 
 
-def verify_installed_asyncio_event_loop(loop_path):
+def verify_installed_asyncio_event_loop(loop_path: str) -> None:
     from twisted.internet import reactor
 
     loop_class = load_object(loop_path)
@@ -181,7 +182,7 @@ def verify_installed_asyncio_event_loop(loop_path):
     )
 
 
-def is_asyncio_reactor_installed():
+def is_asyncio_reactor_installed() -> bool:
     from twisted.internet import reactor
 
     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)

scrapy/utils/request.py

@@ -6,7 +6,18 @@ scrapy.http.Request objects
 import hashlib
 import json
 import warnings
-from typing import Dict, Iterable, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+)
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary
 
@@ -19,11 +30,16 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
 _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
 _deprecated_fingerprint_cache = WeakKeyDictionary()
 
 
-def _serialize_headers(headers, request):
+def _serialize_headers(
+    headers: Iterable[bytes], request: Request
+) -> Generator[bytes, Any, None]:
     for header in headers:
         if header in request.headers:
             yield header
@@ -139,7 +155,7 @@ def request_fingerprint(
     return cache[cache_key]
 
 
-def _request_fingerprint_as_bytes(*args, **kwargs):
+def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         return bytes.fromhex(request_fingerprint(*args, **kwargs))
@@ -231,7 +247,7 @@ class RequestFingerprinter:
     def from_crawler(cls, crawler):
         return cls(crawler)
 
-    def __init__(self, crawler=None):
+    def __init__(self, crawler: Optional["Crawler"] = None):
         if crawler:
             implementation = crawler.settings.get(
                 "REQUEST_FINGERPRINTER_IMPLEMENTATION"
@@ -265,7 +281,7 @@ class RequestFingerprinter:
                 f"and '2.7'."
             )
 
-    def fingerprint(self, request: Request):
+    def fingerprint(self, request: Request) -> bytes:
         return self._fingerprint(request)
 
 
@@ -311,7 +327,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
     If a spider is given, it will try to resolve the callbacks looking at the
     spider for methods with the same name.
     """
-    request_cls = load_object(d["_class"]) if "_class" in d else Request
+    request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request
     kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
     if d.get("callback") and spider:
         kwargs["callback"] = _get_method(spider, d["callback"])
@@ -320,7 +336,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
     return request_cls(**kwargs)
 
 
-def _get_method(obj, name):
+def _get_method(obj: Any, name: Any) -> Any:
     """Helper function for request_from_dict"""
     name = str(name)
     try:

scrapy/utils/serialize.py

@@ -1,6 +1,7 @@
 import datetime
 import decimal
 import json
+from typing import Any
 
 from itemadapter import ItemAdapter, is_item
 from twisted.internet import defer
@@ -12,7 +13,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
     DATE_FORMAT = "%Y-%m-%d"
     TIME_FORMAT = "%H:%M:%S"
 
-    def default(self, o):
+    def default(self, o: Any) -> Any:
         if isinstance(o, set):
             return list(o)
         if isinstance(o, datetime.datetime):

scrapy/utils/template.py

@@ -4,10 +4,10 @@ import re
 import string
 from os import PathLike
 from pathlib import Path
-from typing import Union
+from typing import Any, Union
 
 
-def render_templatefile(path: Union[str, PathLike], **kwargs):
+def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None:
     path_obj = Path(path)
     raw = path_obj.read_text("utf8")
 
@@ -24,7 +24,7 @@ def render_templatefile(path: Union[str, PathLike], **kwargs):
 CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")
 
 
-def string_camelcase(string):
+def string_camelcase(string: str) -> str:
     """Convert a word  to its CamelCase version and remove invalid chars
 
     >>> string_camelcase('lost-pound')

scrapy/utils/test.py

@@ -7,24 +7,30 @@ import os
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
-from unittest import mock
+from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type
+from unittest import TestCase, mock
 
+from twisted.internet.defer import Deferred
 from twisted.trial.unittest import SkipTest
 
+from scrapy import Spider
+from scrapy.crawler import Crawler
 from scrapy.utils.boto import is_botocore_available
 
 
-def assert_gcs_environ():
+def assert_gcs_environ() -> None:
     if "GCS_PROJECT_ID" not in os.environ:
         raise SkipTest("GCS_PROJECT_ID not found")
 
 
-def skip_if_no_boto():
+def skip_if_no_boto() -> None:
     if not is_botocore_available():
         raise SkipTest("missing botocore library")
 
 
-def get_gcs_content_and_delete(bucket, path):
+def get_gcs_content_and_delete(
+    bucket: Any, path: str
+) -> Tuple[bytes, List[Dict[str, str]], Any]:
     from google.cloud import storage
 
     client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
@@ -37,8 +43,13 @@ def get_gcs_content_and_delete(bucket, path):
 
 
 def get_ftp_content_and_delete(
-    path, host, port, username, password, use_active_mode=False
-):
+    path: str,
+    host: str,
+    port: int,
+    username: str,
+    password: str,
+    use_active_mode: bool = False,
+) -> bytes:
     from ftplib import FTP
 
     ftp = FTP()
@@ -46,19 +57,24 @@ def get_ftp_content_and_delete(
     ftp.login(username, password)
     if use_active_mode:
         ftp.set_pasv(False)
-    ftp_data = []
+    ftp_data: List[bytes] = []
 
-    def buffer_data(data):
+    def buffer_data(data: bytes) -> None:
         ftp_data.append(data)
 
     ftp.retrbinary(f"RETR {path}", buffer_data)
     dirname, filename = split(path)
     ftp.cwd(dirname)
     ftp.delete(filename)
-    return "".join(ftp_data)
+    return b"".join(ftp_data)
 
 
-def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True, addons=None):
+def get_crawler(
+    spidercls: Optional[Type[Spider]] = None,
+    settings_dict: Optional[Dict[str, Any]] = None,
+    prevent_warnings: bool = True,
+    addons=None,
+) -> Crawler:
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
     priority.
@@ -82,7 +98,7 @@ def get_pythonpath() -> str:
     return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "")
 
 
-def get_testenv():
+def get_testenv() -> Dict[str, str]:
     """Return a OS environment dict suitable to fork processes that need to import
     this installation of Scrapy, instead of a system installed one.
     """
@@ -91,21 +107,23 @@ def get_testenv():
     return env
 
 
-def assert_samelines(testcase, text1, text2, msg=None):
+def assert_samelines(
+    testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None
+) -> None:
     """Asserts text1 and text2 have the same lines, ignoring differences in
     line endings between platforms
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
 
-def get_from_asyncio_queue(value):
-    q = asyncio.Queue()
+def get_from_asyncio_queue(value: Any) -> Coroutine:
+    q: asyncio.Queue = asyncio.Queue()
     getter = q.get()
     q.put_nowait(value)
     return getter
 
 
-def mock_google_cloud_storage():
+def mock_google_cloud_storage() -> Tuple[Any, Any, Any]:
     """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
     classes and set their proper return values.
     """
@@ -122,7 +140,7 @@ def mock_google_cloud_storage():
     return (client_mock, bucket_mock, blob_mock)
 
 
-def get_web_client_agent_req(url):
+def get_web_client_agent_req(url: str) -> Deferred:
     from twisted.internet import reactor
     from twisted.web.client import Agent  # imports twisted.internet.reactor
 

scrapy/utils/url.py

@@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
 import re
+from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
@@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath  # noqa: F401
 
 from scrapy.utils.python import to_unicode
 
+if TYPE_CHECKING:
+    from scrapy import Spider
 
-def url_is_from_any_domain(url, domains):
+
+UrlT = Union[str, bytes, ParseResult]
+
+
+def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
     """Return True if the url belongs to any of the given domains"""
     host = parse_url(url).netloc.lower()
     if not host:
@@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains):
     return any((host == d) or (host.endswith(f".{d}")) for d in domains)
 
 
-def url_is_from_spider(url, spider):
+def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool:
     """Return True if the url belongs to the given spider"""
     return url_is_from_any_domain(
         url, [spider.name] + list(getattr(spider, "allowed_domains", []))
     )
 
 
-def url_has_any_extension(url, extensions):
+def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
     """Return True if the url ends with one of the extensions provided"""
     lowercase_path = parse_url(url).path.lower()
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
-def parse_url(url, encoding=None):
+def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult:
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
     """
     if isinstance(url, ParseResult):
         return url
-    return urlparse(to_unicode(url, encoding))
+    return cast(ParseResult, urlparse(to_unicode(url, encoding)))
 
 
-def escape_ajax(url):
+def escape_ajax(url: str) -> str:
     """
     Return the crawlable url according to:
     https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
@@ -76,7 +83,7 @@ def escape_ajax(url):
     return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
 
 
-def add_http_if_no_scheme(url):
+def add_http_if_no_scheme(url: str) -> str:
     """Add http as the default scheme if it is missing from the url."""
     match = re.match(r"^\w+://", url, flags=re.I)
     if not match:
@@ -87,7 +94,7 @@ def add_http_if_no_scheme(url):
     return url
 
 
-def _is_posix_path(string):
+def _is_posix_path(string: str) -> bool:
     return bool(
         re.match(
             r"""
@@ -109,7 +116,7 @@ def _is_posix_path(string):
     )
 
 
-def _is_windows_path(string):
+def _is_windows_path(string: str) -> bool:
     return bool(
         re.match(
             r"""
@@ -125,11 +132,11 @@ def _is_windows_path(string):
     )
 
 
-def _is_filesystem_path(string):
+def _is_filesystem_path(string: str) -> bool:
     return _is_posix_path(string) or _is_windows_path(string)
 
 
-def guess_scheme(url):
+def guess_scheme(url: str) -> str:
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
     if _is_filesystem_path(url):
@@ -138,12 +145,12 @@ def guess_scheme(url):
 
 
 def strip_url(
-    url,
-    strip_credentials=True,
-    strip_default_port=True,
-    origin_only=False,
-    strip_fragment=True,
-):
+    url: str,
+    strip_credentials: bool = True,
+    strip_default_port: bool = True,
+    origin_only: bool = False,
+    strip_fragment: bool = True,
+) -> str:
     """Strip URL string from some of its components:
 
     - ``strip_credentials`` removes "user:password@"

scrapy/utils/versions.py

@@ -1,5 +1,6 @@
 import platform
 import sys
+from typing import List, Tuple
 
 import cryptography
 import cssselect
@@ -12,7 +13,7 @@ import scrapy
 from scrapy.utils.ssl import get_openssl_version
 
 
-def scrapy_components_versions():
+def scrapy_components_versions() -> List[Tuple[str, str]]:
     lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
     libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
 

setup.py

@@ -1,26 +1,13 @@
 from pathlib import Path
 
-from pkg_resources import parse_version
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
 version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
 
 
-def has_environment_marker_platform_impl_support():
-    """Code extracted from 'pytest/setup.py'
-    https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31
-
-    The first known release to support environment marker with range operators
-    it is 18.5, see:
-    https://setuptools.readthedocs.io/en/latest/history.html#id235
-    """
-    return parse_version(setuptools_version) >= parse_version("18.5")
-
-
 install_requires = [
     "Twisted>=18.9.0",
-    "cryptography>=3.4.6",
+    "cryptography>=36.0.0",
     "cssselect>=0.9.1",
     "itemloaders>=1.0.1",
     "parsel>=1.5.0",
@@ -34,21 +21,12 @@ install_requires = [
     "setuptools",
     "packaging",
     "tldextract",
-    "lxml>=4.3.0",
+    "lxml>=4.4.1",
 ]
-extras_require = {}
-cpython_dependencies = [
-    "PyDispatcher>=2.0.5",
-]
-if has_environment_marker_platform_impl_support():
-    extras_require[
-        ':platform_python_implementation == "CPython"'
-    ] = cpython_dependencies
-    extras_require[':platform_python_implementation == "PyPy"'] = [
-        "PyPyDispatcher>=2.1.0",
-    ]
-else:
-    install_requires.extend(cpython_dependencies)
+extras_require = {
+    ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],
+    ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],
+}
 
 
 setup(
@@ -80,7 +58,6 @@ setup(
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
@@ -91,7 +68,7 @@ setup(
         "Topic :: Software Development :: Libraries :: Application Frameworks",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=install_requires,
     extras_require=extras_require,
 )

tests/mockserver.py

@@ -1,11 +1,13 @@
 import argparse
 import json
+import os
 import random
 import sys
 from pathlib import Path
 from shutil import rmtree
 from subprocess import PIPE, Popen
 from tempfile import mkdtemp
+from typing import Dict
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
@@ -20,7 +22,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.test import get_testenv
 
 
 def getarg(request, name, default=None, type=None):
@@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None):
     return default
 
 
+def get_mockserver_env() -> Dict[str, str]:
+    """Return a OS environment dict suitable to run mockserver processes."""
+
+    tests_path = Path(__file__).parent.parent
+    pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "")
+    env = os.environ.copy()
+    env["PYTHONPATH"] = pythonpath
+    return env
+
+
 # most of the following resources are copied from twisted.web.test.test_webclient
 class ForeverTakingResource(resource.Resource):
     """
@@ -264,7 +275,7 @@ class MockServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         http_address = self.proc.stdout.readline().strip().decode("ascii")
         https_address = self.proc.stdout.readline().strip().decode("ascii")
@@ -308,7 +319,7 @@ class MockDNSServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         self.host = "127.0.0.1"
         self.port = int(
@@ -331,7 +342,7 @@ class MockFTPServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)],
             stderr=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         for line in self.proc.stderr:
             if b"starting FTP server" in line:

tests/test_cmdline/__init__.py

@@ -1,4 +1,5 @@
 import json
+import os
 import pstats
 import shutil
 import sys
@@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv
 class CmdlineTest(unittest.TestCase):
     def setUp(self):
         self.env = get_testenv()
+        tests_path = Path(__file__).parent.parent
+        self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
         self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
 
     def _execute(self, *new_args, **kwargs):

tests/test_crawler.py

@@ -1,11 +1,12 @@
 import logging
+import os
 import platform
 import subprocess
 import sys
 import warnings
 from pathlib import Path
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted import version as twisted_version
 from twisted.internet import defer
@@ -24,8 +25,8 @@ from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.misc import load_object
 from scrapy.utils.spider import DefaultSpider
-from scrapy.utils.test import get_crawler, get_testenv
-from tests.mockserver import MockServer
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer, get_mockserver_env
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -316,12 +317,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
 class ScriptRunnerMixin:
     script_dir: Path
+    cwd = os.getcwd()
 
     def run_script(self, script_name: str, *script_args):
         script_path = self.script_dir / script_name
         args = [sys.executable, str(script_path)] + list(script_args)
         p = subprocess.Popen(
-            args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            args,
+            env=get_mockserver_env(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         stdout, stderr = p.communicate()
         return stderr.decode("utf-8")

tests/test_downloader_handlers.py

@@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase):
     def test_non_existent(self):
         request = Request(f"file://{self.mktemp()}")
         d = self.download_request(request, Spider("foo"))
-        return self.assertFailure(d, IOError)
+        return self.assertFailure(d, OSError)
 
 
 class ContentLengthHeaderResource(resource.Resource):

tests/test_downloadermiddleware.py

@@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase):
         In particular when some website returns a 30x response with header
         'Content-Encoding: gzip' giving as result the error below:
 
-            exceptions.IOError: Not a gzipped file
+            BadGzipFile: Not a gzipped file (...)
 
         """
         req = Request("http://example.com")
@@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase):
                 "Location": "http://example.com/login",
             },
         )
-        self.assertRaises(IOError, self._download, request=req, response=resp)
+        self.assertRaises(OSError, self._download, request=req, response=resp)
 
 
 class ResponseFromProcessRequestTest(ManagerTestCase):

tests/test_downloadermiddleware_retry.py

@@ -1,5 +1,6 @@
 import logging
 import unittest
+import warnings
 
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
+from scrapy.settings.default_settings import RETRY_EXCEPTIONS
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
@@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase):
             == 2
         )
 
-    def _test_retry_exception(self, req, exception):
+    def test_exception_to_retry_added(self):
+        exc = ValueError
+        settings_dict = {
+            "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc],
+        }
+        crawler = get_crawler(Spider, settings_dict=settings_dict)
+        mw = RetryMiddleware.from_crawler(crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        self._test_retry_exception(req, exc("foo"), mw)
+
+    def test_exception_to_retry_customMiddleware(self):
+        exc = ValueError
+
+        with warnings.catch_warnings(record=True) as warns:
+
+            class MyRetryMiddleware(RetryMiddleware):
+                EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
+
+            self.assertEqual(len(warns), 1)
+
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+
+    def _test_retry_exception(self, req, exception, mw=None):
+        if mw is None:
+            mw = self.mw
+
         # first retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
         # second retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 2)
 
         # discard it
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         self.assertEqual(req, None)
 
 

tests/test_feedexport.py

@@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase):
         with self.assertRaises(NotConfigured):
             FeedExporter.from_crawler(crawler)
 
+    def test_absolute_pathlib_as_uri(self):
+        with tempfile.NamedTemporaryFile(suffix="json") as tmp:
+            settings = {
+                "FEEDS": {
+                    Path(tmp.name).resolve(): {
+                        "format": "json",
+                    },
+                },
+            }
+            crawler = get_crawler(settings_dict=settings)
+            exporter = FeedExporter.from_crawler(crawler)
+            self.assertIsInstance(exporter, FeedExporter)
+
+    def test_relative_pathlib_as_uri(self):
+        settings = {
+            "FEEDS": {
+                Path("./items.json"): {
+                    "format": "json",
+                },
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        exporter = FeedExporter.from_crawler(crawler)
+        self.assertIsInstance(exporter, FeedExporter)
+
 
 class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
     def __init__(self, uri):

tests/test_mail.py

@@ -1,5 +1,3 @@
-# coding=utf-8
-
 import unittest
 from email.charset import Charset
 from io import BytesIO

tests/test_pipeline_files.py

@@ -32,6 +32,7 @@ from scrapy.utils.test import (
     get_gcs_content_and_delete,
     skip_if_no_boto,
 )
+from tests.mockserver import MockFTPServer
 
 from .test_pipeline_media import _mocked_download_func
 
@@ -643,31 +644,29 @@ class TestGCSFilesStore(unittest.TestCase):
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
-        uri = os.environ.get("FTP_TEST_FILE_URI")
-        if not uri:
-            raise unittest.SkipTest("No FTP URI available for testing")
         data = b"TestFTPFilesStore: \xe2\x98\x83"
         buf = BytesIO(data)
         meta = {"foo": "bar"}
         path = "full/filename"
-        store = FTPFilesStore(uri)
-        empty_dict = yield store.stat_file(path, info=None)
-        self.assertEqual(empty_dict, {})
-        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
-        stat = yield store.stat_file(path, info=None)
-        self.assertIn("last_modified", stat)
-        self.assertIn("checksum", stat)
-        self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6")
-        path = f"{store.basedir}/{path}"
-        content = get_ftp_content_and_delete(
-            path,
-            store.host,
-            store.port,
-            store.username,
-            store.password,
-            store.USE_ACTIVE_MODE,
-        )
-        self.assertEqual(data.decode(), content)
+        with MockFTPServer() as ftp_server:
+            store = FTPFilesStore(ftp_server.url("/"))
+            empty_dict = yield store.stat_file(path, info=None)
+            self.assertEqual(empty_dict, {})
+            yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+            stat = yield store.stat_file(path, info=None)
+            self.assertIn("last_modified", stat)
+            self.assertIn("checksum", stat)
+            self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6")
+            path = f"{store.basedir}/{path}"
+            content = get_ftp_content_and_delete(
+                path,
+                store.host,
+                store.port,
+                store.username,
+                store.password,
+                store.USE_ACTIVE_MODE,
+            )
+        self.assertEqual(data, content)
 
 
 class ItemWithFiles(Item):

tests/test_proxy_connect.py

@@ -21,8 +21,6 @@ class MitmProxy:
     auth_pass = "scrapy"
 
     def start(self):
-        from scrapy.utils.test import get_testenv
-
         script = """
 import sys
 from mitmproxy.tools.main import mitmdump
@@ -46,7 +44,6 @@ sys.exit(mitmdump())
                 "--ssl-insecure",
             ],
             stdout=PIPE,
-            env=get_testenv(),
         )
         line = self.proc.stdout.readline().decode("utf-8")
         host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)

tests/test_robotstxt_interface.py

@@ -1,4 +1,3 @@
-# coding=utf-8
 from twisted.trial import unittest
 
 

tests/test_settings/__init__.py

@@ -451,6 +451,35 @@ class SettingsTest(unittest.TestCase):
         self.assertIsInstance(myhandler_instance, FileDownloadHandler)
         self.assertTrue(hasattr(myhandler_instance, "download_request"))
 
+    def test_pop_item_with_default_value(self):
+        settings = Settings()
+
+        with self.assertRaises(KeyError):
+            settings.pop("DUMMY_CONFIG")
+
+        dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value")
+
+        self.assertEqual(
+            repr(dummy_config), "<SettingsAttribute value='dummy_value' priority=20>"
+        )
+        self.assertEqual(dummy_config.value, "dummy_value")
+
+    def test_pop_item_with_immutable_settings(self):
+        settings = Settings(
+            {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
+        )
+
+        self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value")
+
+        settings.freeze()
+
+        with self.assertRaises(TypeError) as error:
+            settings.pop("OTHER_DUMMY_CONFIG")
+
+        self.assertEqual(
+            str(error.exception), "Trying to modify an immutable Settings object"
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_utils_asyncio.py

@@ -4,6 +4,7 @@ from unittest import TestCase
 
 from pytest import mark
 
+from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.reactor import (
     install_reactor,
     is_asyncio_reactor_installed,
@@ -29,6 +30,8 @@ class AsyncioTest(TestCase):
 
         assert original_reactor == reactor
 
+    @mark.only_asyncio()
+    @deferred_f_from_coro_f
     async def test_set_asyncio_event_loop(self):
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
-        assert set_asyncio_event_loop() is asyncio.get_running_loop()
+        assert set_asyncio_event_loop(None) is asyncio.get_running_loop()

tests/test_utils_datatypes.py

@@ -1,9 +1,13 @@
 import copy
 import unittest
+import warnings
 from collections.abc import Mapping, MutableMapping
+from typing import Iterator
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.utils.datatypes import (
+    CaseInsensitiveDict,
     CaselessDict,
     LocalCache,
     LocalWeakReferencedCache,
@@ -14,16 +18,16 @@ from scrapy.utils.python import garbage_collect
 __doctests__ = ["scrapy.utils.datatypes"]
 
 
-class CaselessDictTest(unittest.TestCase):
+class CaseInsensitiveDictMixin:
     def test_init_dict(self):
         seq = {"red": 1, "black": 3}
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
     def test_init_pair_sequence(self):
         seq = (("red", 1), ("black", 3))
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
@@ -42,7 +46,7 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
@@ -67,12 +71,12 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMutableMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d["red"], 1)
         self.assertEqual(d["black"], 3)
 
     def test_caseless(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d["key_Lower"] = 1
         self.assertEqual(d["KEy_loWer"], 1)
         self.assertEqual(d.get("KEy_loWer"), 1)
@@ -82,7 +86,7 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get("key_Lower"), 3)
 
     def test_delete(self):
-        d = CaselessDict({"key_lower": 1})
+        d = self.dict_class({"key_lower": 1})
         del d["key_LOWER"]
         self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
         self.assertRaises(KeyError, d.__getitem__, "key_lower")
@@ -107,15 +111,15 @@ class CaselessDictTest(unittest.TestCase):
     def test_fromkeys(self):
         keys = ("a", "b")
 
-        d = CaselessDict.fromkeys(keys)
+        d = self.dict_class.fromkeys(keys)
         self.assertEqual(d["A"], None)
         self.assertEqual(d["B"], None)
 
-        d = CaselessDict.fromkeys(keys, 1)
+        d = self.dict_class.fromkeys(keys, 1)
         self.assertEqual(d["A"], 1)
         self.assertEqual(d["B"], 1)
 
-        instance = CaselessDict()
+        instance = self.dict_class()
         d = instance.fromkeys(keys)
         self.assertEqual(d["A"], None)
         self.assertEqual(d["B"], None)
@@ -125,31 +129,35 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d["B"], 1)
 
     def test_contains(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d["a"] = 1
-        assert "a" in d
+        assert "A" in d
 
     def test_pop(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d["a"] = 1
         self.assertEqual(d.pop("A"), 1)
         self.assertRaises(KeyError, d.pop, "A")
 
     def test_normkey(self):
-        class MyDict(CaselessDict):
-            def normkey(self, key):
+        class MyDict(self.dict_class):
+            def _normkey(self, key):
                 return key.title()
 
+            normkey = _normkey  # deprecated CaselessDict class
+
         d = MyDict()
         d["key-one"] = 2
         self.assertEqual(list(d.keys()), ["Key-One"])
 
     def test_normvalue(self):
-        class MyDict(CaselessDict):
-            def normvalue(self, value):
+        class MyDict(self.dict_class):
+            def _normvalue(self, value):
                 if value is not None:
                     return value + 1
 
+            normvalue = _normvalue  # deprecated CaselessDict class
+
         d = MyDict({"key": 1})
         self.assertEqual(d["key"], 2)
         self.assertEqual(d.get("key"), 2)
@@ -174,11 +182,51 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get("key"), 2)
 
     def test_copy(self):
-        h1 = CaselessDict({"header1": "value"})
+        h1 = self.dict_class({"header1": "value"})
         h2 = copy.copy(h1)
+        assert isinstance(h2, self.dict_class)
         self.assertEqual(h1, h2)
         self.assertEqual(h1.get("header1"), h2.get("header1"))
-        assert isinstance(h2, CaselessDict)
+        self.assertEqual(h1.get("header1"), h2.get("HEADER1"))
+        h3 = h1.copy()
+        assert isinstance(h3, self.dict_class)
+        self.assertEqual(h1, h3)
+        self.assertEqual(h1.get("header1"), h3.get("header1"))
+        self.assertEqual(h1.get("header1"), h3.get("HEADER1"))
+
+
+class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaseInsensitiveDict
+
+    def test_repr(self):
+        d1 = self.dict_class({"foo": "bar"})
+        self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+        d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        self.assertEqual(
+            repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>"
+        )
+
+    def test_iter(self):
+        d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        iterkeys = iter(d)
+        self.assertIsInstance(iterkeys, Iterator)
+        self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
+
+
+class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaselessDict
+
+    def test_deprecation_message(self):
+        with warnings.catch_warnings(record=True) as caught:
+            self.dict_class({"foo": "bar"})
+
+            self.assertEqual(len(caught), 1)
+            self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+            self.assertEqual(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                str(caught[0].message),
+            )
 
 
 class SequenceExcludeTest(unittest.TestCase):

tests/test_utils_gz.py

@@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_no_gzip_file_raises(self):
         self.assertRaises(
-            IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
+            OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
         )
 
     def test_gunzip_truncated_short(self):

tests/test_utils_iterators.py

@@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
-            self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
+            self.assertTrue(all(isinstance(k, str) for k in result_row.keys()))
+            self.assertTrue(all(isinstance(v, str) for v in result_row.values()))
 
     def test_csviter_delimiter(self):
         body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")

tox.ini

@@ -16,8 +16,6 @@ deps =
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
     mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
-    # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
-    markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -38,10 +36,10 @@ deps =
     mypy==1.2.0
     types-attrs==19.1.0
     types-lxml==2023.3.28
-    types-Pillow==9.4.0.19
-    types-Pygments==2.14.0.7
-    types-pyOpenSSL==23.1.0.1
-    types-setuptools==67.6.0.7
+    types-Pillow==9.5.0.2
+    types-Pygments==2.15.0.0
+    types-pyOpenSSL==23.1.0.2
+    types-setuptools==67.7.0.1
 commands =
     mypy --show-error-codes {posargs: scrapy tests}
 
@@ -71,7 +69,7 @@ commands =
 
 [pinned]
 deps =
-    cryptography==3.4.6
+    cryptography==36.0.0
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
@@ -83,7 +81,7 @@ deps =
     Twisted[http2]==18.9.0
     w3lib==1.17.0
     zope.interface==5.1.0
-    lxml==4.3.0
+    lxml==4.4.1
     -rtests/requirements.txt
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
@@ -96,7 +94,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
 
 [testenv:pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     PyDispatcher==2.0.5
@@ -129,7 +127,7 @@ deps =
     Twisted[http2]
 
 [testenv:extra-deps-pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     boto3==1.20.0
@@ -211,7 +209,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
 
 [testenv:botocore-pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     botocore==1.4.87