From 52d0df5f989903d46e6de4878e1d6a0e87a2c803 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 12 May 2021 13:08:08 -0300
Subject: [PATCH 01/42] CaseInsensitiveDict (deprecate CaselessDict)

---
 scrapy/http/headers.py        | 13 ++++---
 scrapy/pipelines/files.py     |  4 +-
 scrapy/utils/datatypes.py     | 46 +++++++++++++++++++++++
 tests/test_utils_datatypes.py | 71 +++++++++++++++++++++++++----------
 4 files changed, 107 insertions(+), 27 deletions(-)

diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..dfbcf8361 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,5 +1,6 @@
 from w3lib.http import headers_dict_to_raw
-from scrapy.utils.datatypes import CaselessDict
+
+from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
 from scrapy.utils.python import to_unicode
 
 
@@ -76,13 +77,13 @@ class Headers(CaselessDict):
         return headers_dict_to_raw(self)
 
     def to_unicode_dict(self):
-        """ Return headers as a CaselessDict with unicode keys
+        """ Return headers as a CaseInsensitiveDict with unicode keys
         and unicode values. Multiple values are joined with ','.
         """
-        return CaselessDict(
-            (to_unicode(key, encoding=self.encoding),
-             to_unicode(b','.join(value), encoding=self.encoding))
-            for key, value in self.items())
+        return CaseInsensitiveDict(
+            (to_unicode(key, encoding=self.encoding), to_unicode(b','.join(value), encoding=self.encoding))
+            for key, value in self.items()
+        )
 
     def __copy__(self):
         return self.__class__(self)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 13ecd4e6c..2f1a25dfc 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -23,7 +23,7 @@ from scrapy.http import Request
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import md5sum
@@ -143,7 +143,7 @@ class S3FilesStore:
         """ Convert headers to botocore keyword agruments.
         """
         # This is required while we need to support both boto and botocore.
-        mapping = CaselessDict({
+        mapping = CaseInsensitiveDict({
             'Content-Type': 'ContentType',
             'Cache-Control': 'CacheControl',
             'Content-Disposition': 'ContentDisposition',
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index e31284a7f..ca6089e0f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -6,14 +6,30 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import collections
+import warnings
 import weakref
 from collections.abc import Mapping
+from typing import Any, AnyStr
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class CaselessDict(dict):
 
     __slots__ = ()
 
+    def __new__(cls, *args, **kwargs):
+        from scrapy.http.headers import Headers
+
+        if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
+            warnings.warn(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        return super().__new__(cls, *args, **kwargs)
+
     def __init__(self, seq=None):
         super().__init__()
         if seq:
@@ -63,6 +79,36 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
+class CaseInsensitiveDict(collections.UserDict):
+    """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
+
+    It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
+    """
+
+    def __getitem__(self, key: AnyStr) -> Any:
+        return super().__getitem__(self.normkey(key))
+
+    def __setitem__(self, key: AnyStr, value: Any) -> None:
+        super().__setitem__(self.normkey(key), self.normvalue(value))
+
+    def __delitem__(self, key: AnyStr) -> None:
+        super().__delitem__(self.normkey(key))
+
+    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+        return super().__contains__(self.normkey(key))
+
+    def normkey(self, key: AnyStr) -> AnyStr:
+        """Method to normalize dictionary key access"""
+        return key.lower()
+
+    def normvalue(self, value: Any) -> Any:
+        """Method to normalize values prior to be set"""
+        return value
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: {super().__repr__()}>"
+
+
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index e4bccf30e..c033cd537 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,26 +1,34 @@
 import copy
 import unittest
+import warnings
 from collections.abc import Mapping, MutableMapping
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
-from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude
+from scrapy.utils.datatypes import (
+    CaseInsensitiveDict,
+    CaselessDict,
+    LocalCache,
+    LocalWeakReferencedCache,
+    SequenceExclude,
+)
 from scrapy.utils.python import garbage_collect
 
 
 __doctests__ = ['scrapy.utils.datatypes']
 
 
-class CaselessDictTest(unittest.TestCase):
+class CaseInsensitiveDictMixin:
 
     def test_init_dict(self):
         seq = {'red': 1, 'black': 3}
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
     def test_init_pair_sequence(self):
         seq = (('red', 1), ('black', 3))
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
@@ -39,7 +47,7 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
@@ -64,12 +72,12 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMutableMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
     def test_caseless(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d['key_Lower'] = 1
         self.assertEqual(d['KEy_loWer'], 1)
         self.assertEqual(d.get('KEy_loWer'), 1)
@@ -79,19 +87,19 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get('key_Lower'), 3)
 
     def test_delete(self):
-        d = CaselessDict({'key_lower': 1})
+        d = self.dict_class({'key_lower': 1})
         del d['key_LOWER']
         self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
         self.assertRaises(KeyError, d.__getitem__, 'key_lower')
 
     def test_getdefault(self):
-        d = CaselessDict()
+        d = self.dict_class()
         self.assertEqual(d.get('c', 5), 5)
         d['c'] = 10
         self.assertEqual(d.get('c', 5), 10)
 
     def test_setdefault(self):
-        d = CaselessDict({'a': 1, 'b': 2})
+        d = self.dict_class({'a': 1, 'b': 2})
 
         r = d.setdefault('A', 5)
         self.assertEqual(r, 1)
@@ -104,15 +112,15 @@ class CaselessDictTest(unittest.TestCase):
     def test_fromkeys(self):
         keys = ('a', 'b')
 
-        d = CaselessDict.fromkeys(keys)
+        d = self.dict_class.fromkeys(keys)
         self.assertEqual(d['A'], None)
         self.assertEqual(d['B'], None)
 
-        d = CaselessDict.fromkeys(keys, 1)
+        d = self.dict_class.fromkeys(keys, 1)
         self.assertEqual(d['A'], 1)
         self.assertEqual(d['B'], 1)
 
-        instance = CaselessDict()
+        instance = self.dict_class()
         d = instance.fromkeys(keys)
         self.assertEqual(d['A'], None)
         self.assertEqual(d['B'], None)
@@ -122,18 +130,19 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d['B'], 1)
 
     def test_contains(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d['a'] = 1
         assert 'a' in d
+        assert 'A' in d
 
     def test_pop(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d['a'] = 1
         self.assertEqual(d.pop('A'), 1)
         self.assertRaises(KeyError, d.pop, 'A')
 
     def test_normkey(self):
-        class MyDict(CaselessDict):
+        class MyDict(self.dict_class):
             def normkey(self, key):
                 return key.title()
 
@@ -142,7 +151,7 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(list(d.keys()), ['Key-One'])
 
     def test_normvalue(self):
-        class MyDict(CaselessDict):
+        class MyDict(self.dict_class):
             def normvalue(self, value):
                 if value is not None:
                     return value + 1
@@ -171,11 +180,35 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get('key'), 2)
 
     def test_copy(self):
-        h1 = CaselessDict({'header1': 'value'})
+        h1 = self.dict_class({'header1': 'value'})
         h2 = copy.copy(h1)
         self.assertEqual(h1, h2)
         self.assertEqual(h1.get('header1'), h2.get('header1'))
-        assert isinstance(h2, CaselessDict)
+        assert isinstance(h2, self.dict_class)
+
+
+class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaseInsensitiveDict
+
+    def test_repr(self):
+        d = self.dict_class({"foo": "bar"})
+        self.assertEqual(repr(d), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+
+
+class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaselessDict
+
+    def test_deprecation_message(self):
+        with warnings.catch_warnings(record=True) as caught:
+            self.dict_class({"foo": "bar"})
+
+            self.assertEqual(len(caught), 1)
+            self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+            self.assertEqual(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                str(caught[0].message),
+            )
 
 
 class SequenceExcludeTest(unittest.TestCase):

From bbeed6ae8fd9aed3651b104e4cc3e56495e1b1b9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Aug 2021 14:09:30 -0300
Subject: [PATCH 02/42] CaseInsensitiveDict: preserve original keys (only
 lookups are key-insensitive)

---
 scrapy/utils/datatypes.py     | 36 ++++++++++++++++++++++-------------
 tests/test_utils_datatypes.py | 13 +++++++++++--
 2 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index ca6089e0f..1d56811f0 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -79,35 +79,45 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class CaseInsensitiveDict(collections.UserDict):
+class CaseInsensitiveDict(collections.UserDict,):
     """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
 
     It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
     """
 
+    def __init__(self, *args, **kwargs) -> None:
+        self._keys: dict = {}
+        super().__init__(*args, **kwargs)
+
     def __getitem__(self, key: AnyStr) -> Any:
-        return super().__getitem__(self.normkey(key))
+        normalized_key = self.normkey(key)
+        return super().__getitem__(self._keys[normalized_key.lower()])
 
     def __setitem__(self, key: AnyStr, value: Any) -> None:
-        super().__setitem__(self.normkey(key), self.normvalue(value))
+        normalized_key = self.normkey(key)
+        if normalized_key.lower() in self._keys:
+            del self[self._keys[normalized_key.lower()]]
+        super().__setitem__(normalized_key, self.normvalue(value))
+        self._keys[normalized_key.lower()] = normalized_key
 
     def __delitem__(self, key: AnyStr) -> None:
-        super().__delitem__(self.normkey(key))
+        normalized_key = self.normkey(key)
+        stored_key = self._keys.pop(normalized_key.lower())
+        super().__delitem__(stored_key)
 
     def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
-        return super().__contains__(self.normkey(key))
-
-    def normkey(self, key: AnyStr) -> AnyStr:
-        """Method to normalize dictionary key access"""
-        return key.lower()
-
-    def normvalue(self, value: Any) -> Any:
-        """Method to normalize values prior to be set"""
-        return value
+        normalized_key = self.normkey(key)
+        return normalized_key.lower() in self._keys
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
+    def normkey(self, key: AnyStr) -> AnyStr:
+        return key
+
+    def normvalue(self, value: Any) -> Any:
+        return value
+
 
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index c033cd537..5faaabe81 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,4 +1,5 @@
 import copy
+from typing import Iterator
 import unittest
 import warnings
 from collections.abc import Mapping, MutableMapping
@@ -191,8 +192,16 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
     dict_class = CaseInsensitiveDict
 
     def test_repr(self):
-        d = self.dict_class({"foo": "bar"})
-        self.assertEqual(repr(d), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+        d1 = self.dict_class({"foo": "bar"})
+        self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+        d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        self.assertEqual(repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>")
+
+    def test_iter(self):
+        d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        iterkeys = iter(d)
+        self.assertIsInstance(iterkeys, Iterator)
+        self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
 
 
 class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):

From 10ebf6384ed58253c237224d523e602b1f3c2224 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Aug 2021 14:12:55 -0300
Subject: [PATCH 03/42] Remove unnecessary comma

---
 scrapy/utils/datatypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 1d56811f0..6eeabe1ee 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -79,7 +79,7 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class CaseInsensitiveDict(collections.UserDict,):
+class CaseInsensitiveDict(collections.UserDict):
     """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
 
     It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.

From 1c031b8a8dd719e6011ee29889bc8181cdbc9a9b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 12 May 2022 13:10:08 -0300
Subject: [PATCH 04/42] Underscore CaseInsensitiveDict normkey/normvalue

---
 scrapy/utils/datatypes.py     | 19 +++++++++----------
 tests/test_utils_datatypes.py | 10 +++++++---
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index f45e1c9b8..807a95504 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -80,9 +80,8 @@ class CaselessDict(dict):
 
 
 class CaseInsensitiveDict(collections.UserDict):
-    """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
-
-    It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
+    """A dict-like structure that accepts strings or bytes
+    as keys and allows case-insensitive lookups.
     """
 
     def __init__(self, *args, **kwargs) -> None:
@@ -90,32 +89,32 @@ class CaseInsensitiveDict(collections.UserDict):
         super().__init__(*args, **kwargs)
 
     def __getitem__(self, key: AnyStr) -> Any:
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         return super().__getitem__(self._keys[normalized_key.lower()])
 
     def __setitem__(self, key: AnyStr, value: Any) -> None:
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         if normalized_key.lower() in self._keys:
             del self[self._keys[normalized_key.lower()]]
-        super().__setitem__(normalized_key, self.normvalue(value))
+        super().__setitem__(normalized_key, self._normvalue(value))
         self._keys[normalized_key.lower()] = normalized_key
 
     def __delitem__(self, key: AnyStr) -> None:
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         stored_key = self._keys.pop(normalized_key.lower())
         super().__delitem__(stored_key)
 
     def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         return normalized_key.lower() in self._keys
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
-    def normkey(self, key: AnyStr) -> AnyStr:
+    def _normkey(self, key: AnyStr) -> AnyStr:
         return key
 
-    def normvalue(self, value: Any) -> Any:
+    def _normvalue(self, value: Any) -> Any:
         return value
 
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 5faaabe81..0a724f237 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,8 +1,8 @@
 import copy
-from typing import Iterator
 import unittest
 import warnings
 from collections.abc import Mapping, MutableMapping
+from typing import Iterator
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
@@ -144,19 +144,23 @@ class CaseInsensitiveDictMixin:
 
     def test_normkey(self):
         class MyDict(self.dict_class):
-            def normkey(self, key):
+            def _normkey(self, key):
                 return key.title()
 
+            normkey = _normkey  # deprecated CaselessDict class
+
         d = MyDict()
         d['key-one'] = 2
         self.assertEqual(list(d.keys()), ['Key-One'])
 
     def test_normvalue(self):
         class MyDict(self.dict_class):
-            def normvalue(self, value):
+            def _normvalue(self, value):
                 if value is not None:
                     return value + 1
 
+            normvalue = _normvalue  # deprecated CaselessDict class
+
         d = MyDict({'key': 1})
         self.assertEqual(d['key'], 2)
         self.assertEqual(d.get('key'), 2)

From 2c65066ad9e293630da2c594af06ad483abe800d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 27 May 2022 19:56:42 -0300
Subject: [PATCH 05/42] Avoid exceptions on copy

---
 scrapy/utils/datatypes.py     | 7 +++++--
 tests/test_utils_datatypes.py | 8 +++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 807a95504..fd5ac3b08 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -94,8 +94,11 @@ class CaseInsensitiveDict(collections.UserDict):
 
     def __setitem__(self, key: AnyStr, value: Any) -> None:
         normalized_key = self._normkey(key)
-        if normalized_key.lower() in self._keys:
-            del self[self._keys[normalized_key.lower()]]
+        try:
+            lower_key = self._keys[normalized_key.lower()]
+            del self[lower_key]
+        except KeyError:
+            pass
         super().__setitem__(normalized_key, self._normvalue(value))
         self._keys[normalized_key.lower()] = normalized_key
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 0a724f237..36df9006f 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -187,9 +187,15 @@ class CaseInsensitiveDictMixin:
     def test_copy(self):
         h1 = self.dict_class({'header1': 'value'})
         h2 = copy.copy(h1)
+        assert isinstance(h2, self.dict_class)
         self.assertEqual(h1, h2)
         self.assertEqual(h1.get('header1'), h2.get('header1'))
-        assert isinstance(h2, self.dict_class)
+        self.assertEqual(h1.get('header1'), h2.get('HEADER1'))
+        h3 = h1.copy()
+        assert isinstance(h3, self.dict_class)
+        self.assertEqual(h1, h3)
+        self.assertEqual(h1.get('header1'), h3.get('header1'))
+        self.assertEqual(h1.get('header1'), h3.get('HEADER1'))
 
 
 class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):

From 4596a58a1333b8174b09336ab74c32c2e14c40f4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 16 Apr 2023 01:12:21 +0400
Subject: [PATCH 06/42] Typing for smaller scrapy/utils/ modules.

---
 scrapy/utils/boto.py       |  2 +-
 scrapy/utils/decorators.py | 16 +++++++++-------
 scrapy/utils/serialize.py  |  3 ++-
 scrapy/utils/versions.py   |  3 ++-
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 085ee7d25..53cfeddd0 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,7 @@
 """Boto/botocore helpers"""
 
 
-def is_botocore_available():
+def is_botocore_available() -> bool:
     try:
         import botocore  # noqa: F401
 
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 4e684645b..04186559f 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -1,19 +1,21 @@
 import warnings
 from functools import wraps
+from typing import Any, Callable
 
 from twisted.internet import defer, threads
+from twisted.internet.defer import Deferred
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def deprecated(use_instead=None):
+def deprecated(use_instead: Any = None) -> Callable:
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used."""
 
-    def deco(func):
+    def deco(func: Callable) -> Callable:
         @wraps(func)
-        def wrapped(*args, **kwargs):
+        def wrapped(*args: Any, **kwargs: Any) -> Any:
             message = f"Call to deprecated function {func.__name__}."
             if use_instead:
                 message += f" Use {use_instead} instead."
@@ -28,23 +30,23 @@ def deprecated(use_instead=None):
     return deco
 
 
-def defers(func):
+def defers(func: Callable) -> Callable[..., Deferred]:
     """Decorator to make sure a function always returns a deferred"""
 
     @wraps(func)
-    def wrapped(*a, **kw):
+    def wrapped(*a: Any, **kw: Any) -> Deferred:
         return defer.maybeDeferred(func, *a, **kw)
 
     return wrapped
 
 
-def inthread(func):
+def inthread(func: Callable) -> Callable[..., Deferred]:
     """Decorator to call a function in a thread and return a deferred with the
     result
     """
 
     @wraps(func)
-    def wrapped(*a, **kw):
+    def wrapped(*a: Any, **kw: Any) -> Deferred:
         return threads.deferToThread(func, *a, **kw)
 
     return wrapped
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 414658944..3b4f67f00 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -1,6 +1,7 @@
 import datetime
 import decimal
 import json
+from typing import Any
 
 from itemadapter import ItemAdapter, is_item
 from twisted.internet import defer
@@ -12,7 +13,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
     DATE_FORMAT = "%Y-%m-%d"
     TIME_FORMAT = "%H:%M:%S"
 
-    def default(self, o):
+    def default(self, o: Any) -> Any:
         if isinstance(o, set):
             return list(o)
         if isinstance(o, datetime.datetime):
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index b0737d3d5..9b637bdb0 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -1,5 +1,6 @@
 import platform
 import sys
+from typing import List, Tuple
 
 import cryptography
 import cssselect
@@ -12,7 +13,7 @@ import scrapy
 from scrapy.utils.ssl import get_openssl_version
 
 
-def scrapy_components_versions():
+def scrapy_components_versions() -> List[Tuple[str, str]]:
     lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
     libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
 

From e0dbc83bd269e256e8247525e4d5a1d07658b635 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 16 Apr 2023 01:30:04 +0400
Subject: [PATCH 07/42] More typing for scrapy/utils/request.py.

---
 scrapy/utils/request.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 6d8be991d..6c7f3b345 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -6,7 +6,18 @@ scrapy.http.Request objects
 import hashlib
 import json
 import warnings
-from typing import Dict, Iterable, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+)
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary
 
@@ -19,11 +30,16 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
 _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
 _deprecated_fingerprint_cache = WeakKeyDictionary()
 
 
-def _serialize_headers(headers, request):
+def _serialize_headers(
+    headers: Iterable[bytes], request: Request
+) -> Generator[bytes, Any, None]:
     for header in headers:
         if header in request.headers:
             yield header
@@ -139,7 +155,7 @@ def request_fingerprint(
     return cache[cache_key]
 
 
-def _request_fingerprint_as_bytes(*args, **kwargs):
+def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         return bytes.fromhex(request_fingerprint(*args, **kwargs))
@@ -231,7 +247,7 @@ class RequestFingerprinter:
     def from_crawler(cls, crawler):
         return cls(crawler)
 
-    def __init__(self, crawler=None):
+    def __init__(self, crawler: Optional["Crawler"] = None):
         if crawler:
             implementation = crawler.settings.get(
                 "REQUEST_FINGERPRINTER_IMPLEMENTATION"
@@ -265,7 +281,7 @@ class RequestFingerprinter:
                 f"and '2.7'."
             )
 
-    def fingerprint(self, request: Request):
+    def fingerprint(self, request: Request) -> bytes:
         return self._fingerprint(request)
 
 
@@ -311,7 +327,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
     If a spider is given, it will try to resolve the callbacks looking at the
     spider for methods with the same name.
     """
-    request_cls = load_object(d["_class"]) if "_class" in d else Request
+    request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request
     kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
     if d.get("callback") and spider:
         kwargs["callback"] = _get_method(spider, d["callback"])
@@ -320,7 +336,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
     return request_cls(**kwargs)
 
 
-def _get_method(obj, name):
+def _get_method(obj: Any, name: Any) -> Any:
     """Helper function for request_from_dict"""
     name = str(name)
     try:

From f64a7dedca6d4bf33f86c633a0381a8017eb79f7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 23 Apr 2023 22:56:27 +0400
Subject: [PATCH 08/42] Add typing to scrapy/utils/url.py.

---
 scrapy/utils/url.py | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 833aa3e20..22b4197f9 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
 import re
+from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
@@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath  # noqa: F401
 
 from scrapy.utils.python import to_unicode
 
+if TYPE_CHECKING:
+    from scrapy import Spider
 
-def url_is_from_any_domain(url, domains):
+
+UrlT = Union[str, bytes, ParseResult]
+
+
+def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
     """Return True if the url belongs to any of the given domains"""
     host = parse_url(url).netloc.lower()
     if not host:
@@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains):
     return any((host == d) or (host.endswith(f".{d}")) for d in domains)
 
 
-def url_is_from_spider(url, spider):
+def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool:
     """Return True if the url belongs to the given spider"""
     return url_is_from_any_domain(
         url, [spider.name] + list(getattr(spider, "allowed_domains", []))
     )
 
 
-def url_has_any_extension(url, extensions):
+def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
     """Return True if the url ends with one of the extensions provided"""
     lowercase_path = parse_url(url).path.lower()
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
-def parse_url(url, encoding=None):
+def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult:
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
     """
     if isinstance(url, ParseResult):
         return url
-    return urlparse(to_unicode(url, encoding))
+    return cast(ParseResult, urlparse(to_unicode(url, encoding)))
 
 
-def escape_ajax(url):
+def escape_ajax(url: str) -> str:
     """
     Return the crawlable url according to:
     https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
@@ -76,7 +83,7 @@ def escape_ajax(url):
     return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
 
 
-def add_http_if_no_scheme(url):
+def add_http_if_no_scheme(url: str) -> str:
     """Add http as the default scheme if it is missing from the url."""
     match = re.match(r"^\w+://", url, flags=re.I)
     if not match:
@@ -87,7 +94,7 @@ def add_http_if_no_scheme(url):
     return url
 
 
-def _is_posix_path(string):
+def _is_posix_path(string: str) -> bool:
     return bool(
         re.match(
             r"""
@@ -109,7 +116,7 @@ def _is_posix_path(string):
     )
 
 
-def _is_windows_path(string):
+def _is_windows_path(string: str) -> bool:
     return bool(
         re.match(
             r"""
@@ -125,11 +132,11 @@ def _is_windows_path(string):
     )
 
 
-def _is_filesystem_path(string):
+def _is_filesystem_path(string: str) -> bool:
     return _is_posix_path(string) or _is_windows_path(string)
 
 
-def guess_scheme(url):
+def guess_scheme(url: str) -> str:
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
     if _is_filesystem_path(url):
@@ -138,12 +145,12 @@ def guess_scheme(url):
 
 
 def strip_url(
-    url,
-    strip_credentials=True,
-    strip_default_port=True,
-    origin_only=False,
-    strip_fragment=True,
-):
+    url: str,
+    strip_credentials: bool = True,
+    strip_default_port: bool = True,
+    origin_only: bool = False,
+    strip_fragment: bool = True,
+) -> str:
     """Strip URL string from some of its components:
 
     - ``strip_credentials`` removes "user:password@"

From 7347d021457866ade62a6ed8a0839766e91032e4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 01:12:52 +0400
Subject: [PATCH 09/42] Add typing to scrapy/utils/datatypes.py.

---
 scrapy/resolver.py        |  8 +++++---
 scrapy/utils/datatypes.py | 28 ++++++++++++++++------------
 scrapy/utils/misc.py      | 12 +++++++-----
 3 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 6cbe01cbf..e2e8beff4 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,3 +1,5 @@
+from typing import Any
+
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
 from twisted.internet.interfaces import (
@@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider
 from scrapy.utils.datatypes import LocalCache
 
 # TODO: cache misses
-dnscache = LocalCache(10000)
+dnscache: LocalCache[str, Any] = LocalCache(10000)
 
 
 @implementer(IResolverSimple)
@@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver):
     def install_on_reactor(self):
         self.reactor.installResolver(self)
 
-    def getHostByName(self, name, timeout=None):
+    def getHostByName(self, name: str, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
         # in Twisted<=16.6, getHostByName() is always called with
@@ -110,7 +112,7 @@ class CachingHostnameResolver:
     def resolveHostName(
         self,
         resolutionReceiver,
-        hostName,
+        hostName: str,
         portNumber=0,
         addressTypes=None,
         transportSemantics="TCP",
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index fa57a4f26..599b201ea 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -8,6 +8,10 @@ This module must not depend on any module outside the Standard Library.
 import collections
 import weakref
 from collections.abc import Mapping
+from typing import Any, Optional, Sequence, TypeVar
+
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
 
 
 class CaselessDict(dict):
@@ -64,24 +68,24 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class LocalCache(collections.OrderedDict):
+class LocalCache(collections.OrderedDict[_KT, _VT]):
     """Dictionary with a finite number of keys.
 
     Older items expires first.
     """
 
-    def __init__(self, limit=None):
+    def __init__(self, limit: Optional[int] = None):
         super().__init__()
-        self.limit = limit
+        self.limit: Optional[int] = limit
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: _KT, value: _VT) -> None:
         if self.limit:
             while len(self) >= self.limit:
                 self.popitem(last=False)
         super().__setitem__(key, value)
 
 
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]):
     """
     A weakref.WeakKeyDictionary implementation that uses LocalCache as its
     underlying data structure, making it ordered and capable of being size-limited.
@@ -93,17 +97,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     it cannot be instantiated with an initial dictionary.
     """
 
-    def __init__(self, limit=None):
+    def __init__(self, limit: Optional[int] = None):
         super().__init__()
-        self.data = LocalCache(limit=limit)
+        self.data: LocalCache = LocalCache(limit=limit)
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: _KT, value: _VT) -> None:
         try:
             super().__setitem__(key, value)
         except TypeError:
             pass  # key is not weak-referenceable, skip caching
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: _KT) -> Optional[_VT]:  # type: ignore[override]
         try:
             return super().__getitem__(key)
         except (TypeError, KeyError):
@@ -113,8 +117,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 class SequenceExclude:
     """Object to test if an item is NOT within some sequence."""
 
-    def __init__(self, seq):
-        self.seq = seq
+    def __init__(self, seq: Sequence):
+        self.seq: Sequence = seq
 
-    def __contains__(self, item):
+    def __contains__(self, item: Any) -> bool:
         return item not in self.seq
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index d861c9ab6..ea3f934c7 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -214,16 +214,18 @@ def walk_callable(node):
         yield node
 
 
-_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
+_generator_callbacks_cache: LocalWeakReferencedCache[
+    Callable, bool
+] = LocalWeakReferencedCache(limit=128)
 
 
-def is_generator_with_return_value(callable):
+def is_generator_with_return_value(callable: Callable) -> bool:
     """
     Returns True if a callable is a generator function which includes a
     'return' statement with a value different than None, False otherwise
     """
     if callable in _generator_callbacks_cache:
-        return _generator_callbacks_cache[callable]
+        return bool(_generator_callbacks_cache[callable])
 
     def returns_none(return_node):
         value = return_node.value
@@ -248,10 +250,10 @@ def is_generator_with_return_value(callable):
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
                 _generator_callbacks_cache[callable] = True
-                return _generator_callbacks_cache[callable]
+                return bool(_generator_callbacks_cache[callable])
 
     _generator_callbacks_cache[callable] = False
-    return _generator_callbacks_cache[callable]
+    return bool(_generator_callbacks_cache[callable])
 
 
 def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:

From ea299dfd7ce8766c2c9430fe8b297fddad45adee Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 01:21:00 +0400
Subject: [PATCH 10/42] Add typing to scrapy/utils/misc.py.

---
 scrapy/utils/misc.py   | 46 +++++++++++++++++++++++++++++-------------
 scrapy/utils/python.py |  4 ++--
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index ea3f934c7..defc8663d 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -10,7 +10,21 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from typing import TYPE_CHECKING, Any, Callable, Union
+from types import ModuleType
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Deque,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Pattern,
+    Union,
+    cast,
+)
 
 from w3lib.html import replace_entities
 
@@ -26,7 +40,7 @@ if TYPE_CHECKING:
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 
 
-def arg_to_iter(arg):
+def arg_to_iter(arg: Any) -> Iterable[Any]:
     """Convert an argument to an iterable. The argument can be a None, single
     value, or an iterable.
 
@@ -35,7 +49,7 @@ def arg_to_iter(arg):
     if arg is None:
         return []
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return arg
+        return cast(Iterable[Any], arg)
     return [arg]
 
 
@@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any:
     return obj
 
 
-def walk_modules(path):
+def walk_modules(path: str) -> List[ModuleType]:
     """Loads a module and all its submodules from the given module path and
     returns them. If *any* module throws an exception while importing, that
     exception is thrown back.
@@ -80,7 +94,7 @@ def walk_modules(path):
     For example: walk_modules('scrapy.utils')
     """
 
-    mods = []
+    mods: List[ModuleType] = []
     mod = import_module(path)
     mods.append(mod)
     if hasattr(mod, "__path__"):
@@ -94,7 +108,9 @@ def walk_modules(path):
     return mods
 
 
-def extract_regex(regex, text, encoding="utf-8"):
+def extract_regex(
+    regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
+) -> List[str]:
     """Extract a list of unicode strings from the given text/encoding using the following policies:
 
     * if the regex contains a named group called "extract" that will be returned
@@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"):
         regex = re.compile(regex, re.UNICODE)
 
     try:
-        strings = [regex.search(text).group("extract")]  # named group
+        # named group
+        strings = [regex.search(text).group("extract")]  # type: ignore[union-attr]
     except Exception:
-        strings = regex.findall(text)  # full regex or numbered groups
+        # full regex or numbered groups
+        strings = regex.findall(text)
     strings = flatten(strings)
 
     if isinstance(text, str):
@@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"):
     ]
 
 
-def md5sum(file):
+def md5sum(file: IO) -> str:
     """Calculate the md5 checksum of a file-like object without reading its
     whole content in memory.
 
@@ -140,7 +158,7 @@ def md5sum(file):
     return m.hexdigest()
 
 
-def rel_has_nofollow(rel):
+def rel_has_nofollow(rel: Optional[str]) -> bool:
     """Return True if link rel attribute has nofollow type"""
     return rel is not None and "nofollow" in rel.replace(",", " ").split()
 
@@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
 
 
 @contextmanager
-def set_environ(**kwargs):
+def set_environ(**kwargs: str) -> Generator[None, Any, None]:
     """Temporarily set environment variables inside the context manager and
     fully restore previous environment afterwards
     """
@@ -198,11 +216,11 @@ def set_environ(**kwargs):
                 os.environ[k] = v
 
 
-def walk_callable(node):
+def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
     """Similar to ``ast.walk``, but walks only function body and skips nested
     functions defined within the node.
     """
-    todo = deque([node])
+    todo: Deque[ast.AST] = deque([node])
     walked_func_def = False
     while todo:
         node = todo.popleft()
@@ -227,7 +245,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:
     if callable in _generator_callbacks_cache:
         return bool(_generator_callbacks_cache[callable])
 
-    def returns_none(return_node):
+    def returns_none(return_node: ast.Return) -> bool:
         value = return_node.value
         return (
             value is None or isinstance(value, ast.NameConstant) and value.value is None
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 27816c0df..ae8feaf7d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -22,7 +22,7 @@ from typing import (
 from scrapy.utils.asyncgen import as_async_generator
 
 
-def flatten(x):
+def flatten(x: Iterable) -> list:
     """flatten(sequence) -> list
 
     Returns a single, flat list which contains all elements retrieved
@@ -42,7 +42,7 @@ def flatten(x):
     return list(iflatten(x))
 
 
-def iflatten(x):
+def iflatten(x: Iterable) -> Iterable:
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""

From 43ee483a0dc6c7883ba9f219ac8b4fd95647b83d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 01:34:54 +0400
Subject: [PATCH 11/42] Add typing to scrapy/utils/reactor.py.

---
 scrapy/utils/reactor.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index f1b9239e6..ad3d1d8bc 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,7 +1,8 @@
 import asyncio
 import sys
+from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
 from contextlib import suppress
-from typing import Any, Callable, Dict, Optional, Sequence
+from typing import Any, Callable, Dict, Optional, Sequence, Type
 from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
@@ -57,7 +58,7 @@ class CallLaterOnce:
         return self._func(*self._a, **self._kw)
 
 
-def set_asyncio_event_loop_policy():
+def set_asyncio_event_loop_policy() -> None:
     """The policy functions from asyncio often behave unexpectedly,
     so we restrict their use to the absolutely essential case.
     This should only be used to install the reactor.
@@ -65,7 +66,7 @@ def set_asyncio_event_loop_policy():
     _get_asyncio_event_loop_policy()
 
 
-def get_asyncio_event_loop_policy():
+def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
     warn(
         "Call to deprecated function "
         "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
@@ -81,7 +82,7 @@ def get_asyncio_event_loop_policy():
     return _get_asyncio_event_loop_policy()
 
 
-def _get_asyncio_event_loop_policy():
+def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
     policy = asyncio.get_event_loop_policy()
     if (
         sys.version_info >= (3, 8)
@@ -93,7 +94,7 @@ def _get_asyncio_event_loop_policy():
     return policy
 
 
-def install_reactor(reactor_path, event_loop_path=None):
+def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None:
     """Installs the :mod:`~twisted.internet.reactor` with the specified
     import path. Also installs the asyncio event loop with the specified import
     path if the asyncio reactor is enabled"""
@@ -111,14 +112,14 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
-def _get_asyncio_event_loop():
+def _get_asyncio_event_loop() -> AbstractEventLoop:
     return set_asyncio_event_loop(None)
 
 
-def set_asyncio_event_loop(event_loop_path):
+def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop:
     """Sets and returns the event loop with specified import path."""
     if event_loop_path is not None:
-        event_loop_class = load_object(event_loop_path)
+        event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path)
         event_loop = event_loop_class()
         asyncio.set_event_loop(event_loop)
     else:
@@ -146,7 +147,7 @@ def set_asyncio_event_loop(event_loop_path):
     return event_loop
 
 
-def verify_installed_reactor(reactor_path):
+def verify_installed_reactor(reactor_path: str) -> None:
     """Raises :exc:`Exception` if the installed
     :mod:`~twisted.internet.reactor` does not match the specified import
     path."""
@@ -162,7 +163,7 @@ def verify_installed_reactor(reactor_path):
         raise Exception(msg)
 
 
-def verify_installed_asyncio_event_loop(loop_path):
+def verify_installed_asyncio_event_loop(loop_path: str) -> None:
     from twisted.internet import reactor
 
     loop_class = load_object(loop_path)
@@ -181,7 +182,7 @@ def verify_installed_asyncio_event_loop(loop_path):
     )
 
 
-def is_asyncio_reactor_installed():
+def is_asyncio_reactor_installed() -> bool:
     from twisted.internet import reactor
 
     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)

From 4da86915109f77066c306130ecd122a17430191d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 02:02:33 +0400
Subject: [PATCH 12/42] Add typing to scrapy/utils/gz.py and remove dead code.

---
 scrapy/utils/gz.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index e5df34d2e..98ca510ed 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,15 +1,18 @@
 import struct
 from gzip import GzipFile
 from io import BytesIO
+from typing import List
+
+from scrapy.http import Response
 
 
-def gunzip(data):
+def gunzip(data: bytes) -> bytes:
     """Gunzip the given data and return as much data as possible.
 
     This is resilient to CRC checksum errors.
     """
     f = GzipFile(fileobj=BytesIO(data))
-    output_list = []
+    output_list: List[bytes] = []
     chunk = b"."
     while chunk:
         try:
@@ -18,17 +21,13 @@ def gunzip(data):
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
-            # some pages are quite small so output_list is empty and f.extrabuf
-            # contains the whole page content
-            if output_list or getattr(f, "extrabuf", None):
-                try:
-                    output_list.append(f.extrabuf[-f.extrasize :])
-                finally:
-                    break
+            # some pages are quite small so output_list is empty
+            if output_list:
+                break
             else:
                 raise
     return b"".join(output_list)
 
 
-def gzip_magic_number(response):
+def gzip_magic_number(response: Response) -> bool:
     return response.body[:3] == b"\x1f\x8b\x08"

From d400f1ac0664768d04985f0a00bc6d47a54957fe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 18:35:24 +0400
Subject: [PATCH 13/42] Add more typing to scrapy/utils/python.py.

---
 scrapy/utils/python.py | 58 ++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index ae8feaf7d..0b5dc324f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+import collections.abc
 import gc
 import inspect
 import re
@@ -12,9 +13,17 @@ from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
+    AsyncIterator,
+    Callable,
+    Dict,
+    Generator,
     Iterable,
+    Iterator,
+    List,
     Mapping,
     Optional,
+    Pattern,
+    Tuple,
     Union,
     overload,
 )
@@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool:
     return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
-def unique(list_, key=lambda x: x):
+def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list:
     """efficient function to uniquify a list preserving item order"""
     seen = set()
     result = []
@@ -124,7 +133,9 @@ def to_bytes(
     return text.encode(encoding, errors)
 
 
-def re_rsearch(pattern, text, chunk_size=1024):
+def re_rsearch(
+    pattern: Union[str, Pattern], text: str, chunk_size: int = 1024
+) -> Optional[Tuple[int, int]]:
     """
     This function does a reverse search in a text using a regular expression
     given in the attribute 'pattern'.
@@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter():
+    def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
         offset = len(text)
         while True:
             offset -= chunk_size * 1024
@@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024):
     return None
 
 
-def memoizemethod_noargs(method):
+def memoizemethod_noargs(method: Callable) -> Callable:
     """Decorator to cache the result of a method (without arguments) using a
     weak reference to its object
     """
-    cache = weakref.WeakKeyDictionary()
+    cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary()
 
     @wraps(method)
-    def new_method(self, *args, **kwargs):
+    def new_method(self: Any, *args: Any, **kwargs: Any) -> Any:
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
         return cache[self]
@@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool:
     return all(c not in _BINARYCHARS for c in data)
 
 
-def get_func_args(func, stripself=False):
+def get_func_args(func: Callable, stripself: bool = False) -> List[str]:
     """Return the argument name list of a callable object"""
     if not callable(func):
         raise TypeError(f"func must be callable, got '{type(func).__name__}'")
 
-    args = []
+    args: List[str] = []
     try:
         sig = inspect.signature(func)
     except ValueError:
@@ -217,7 +228,7 @@ def get_func_args(func, stripself=False):
     return args
 
 
-def get_spec(func):
+def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
     """Returns (args, kwargs) tuple for a function
     >>> import re
     >>> get_spec(re.match)
@@ -246,7 +257,7 @@ def get_spec(func):
     else:
         raise TypeError(f"{type(func)} is not callable")
 
-    defaults = spec.defaults or []
+    defaults: Tuple[Any, ...] = spec.defaults or ()
 
     firstdefault = len(spec.args) - len(defaults)
     args = spec.args[:firstdefault]
@@ -254,7 +265,9 @@ def get_spec(func):
     return args, kwargs
 
 
-def equal_attributes(obj1, obj2, attributes):
+def equal_attributes(
+    obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]
+) -> bool:
     """Compare two objects attributes"""
     # not attributes given return False by default
     if not attributes:
@@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable:
     ...
 
 
-def without_none_values(iterable):
+def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:
     """Return a copy of ``iterable`` with all ``None`` entries removed.
 
     If ``iterable`` is a mapping, return a dictionary where all pairs that have
     value ``None`` have been removed.
     """
-    try:
+    if isinstance(iterable, collections.abc.Mapping):
         return {k: v for k, v in iterable.items() if v is not None}
-    except AttributeError:
-        return type(iterable)((v for v in iterable if v is not None))
+    else:
+        # the iterable __init__ must take another iterable
+        return type(iterable)(v for v in iterable if v is not None)  # type: ignore[call-arg]
 
 
-def global_object_name(obj):
+def global_object_name(obj: Any) -> str:
     """
     Return full name of a global object.
 
@@ -307,14 +321,14 @@ def global_object_name(obj):
 
 if hasattr(sys, "pypy_version_info"):
 
-    def garbage_collect():
+    def garbage_collect() -> None:
         # Collecting weakreferences can take two collections on PyPy.
         gc.collect()
         gc.collect()
 
 else:
 
-    def garbage_collect():
+    def garbage_collect() -> None:
         gc.collect()
 
 
@@ -329,10 +343,10 @@ class MutableChain(Iterable):
     def extend(self, *iterables: Iterable) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator:
         return self
 
-    def __next__(self):
+    def __next__(self) -> Any:
         return next(self.data)
 
 
@@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable):
     def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
         self.data = _async_chain(self.data, _async_chain(*iterables))
 
-    def __aiter__(self):
+    def __aiter__(self) -> AsyncIterator:
         return self
 
-    async def __anext__(self):
+    async def __anext__(self) -> Any:
         return await self.data.__anext__()

From 9661a5c4913e4be314572e12a49c215e7631db88 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 18:50:23 +0400
Subject: [PATCH 14/42] Add typing to scrapy/utils/deprecate.py.

---
 scrapy/utils/deprecate.py | 59 ++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index f4d6e0451..ab2719bb3 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -2,12 +2,12 @@
 
 import inspect
 import warnings
-from typing import List, Tuple
+from typing import Any, List, Optional, Tuple, Type, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def attribute(obj, oldattr, newattr, version="0.12"):
+def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None:
     cname = obj.__class__.__name__
     warnings.warn(
         f"{cname}.{oldattr} attribute is deprecated and will be no longer supported "
@@ -18,16 +18,16 @@ def attribute(obj, oldattr, newattr, version="0.12"):
 
 
 def create_deprecated_class(
-    name,
-    new_class,
-    clsdict=None,
-    warn_category=ScrapyDeprecationWarning,
-    warn_once=True,
-    old_class_path=None,
-    new_class_path=None,
-    subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
-    instance_warn_message="{cls} is deprecated, instantiate {new} instead.",
-):
+    name: str,
+    new_class: type,
+    clsdict: Optional[dict[str, Any]] = None,
+    warn_category: Type[Warning] = ScrapyDeprecationWarning,
+    warn_once: bool = True,
+    old_class_path: Optional[str] = None,
+    new_class_path: Optional[str] = None,
+    subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.",
+    instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.",
+) -> type:
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
     Subclasses of ``new_class`` are considered subclasses of this class.
@@ -53,17 +53,20 @@ def create_deprecated_class(
     OldName.
     """
 
-    class DeprecatedClass(new_class.__class__):
-        deprecated_class = None
-        warned_on_subclass = False
+    # https://github.com/python/mypy/issues/4177
+    class DeprecatedClass(new_class.__class__):  # type: ignore[misc, name-defined]
+        deprecated_class: Optional[type] = None
+        warned_on_subclass: bool = False
 
-        def __new__(metacls, name, bases, clsdict_):
+        def __new__(
+            metacls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]
+        ) -> type:
             cls = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
 
-        def __init__(cls, name, bases, clsdict_):
+        def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]):
             meta = cls.__class__
             old = meta.deprecated_class
             if old in bases and not (warn_once and meta.warned_on_subclass):
@@ -81,10 +84,10 @@ def create_deprecated_class(
         # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
         # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
         # for implementation details
-        def __instancecheck__(cls, inst):
+        def __instancecheck__(cls, inst: Any) -> bool:
             return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__))
 
-        def __subclasscheck__(cls, sub):
+        def __subclasscheck__(cls, sub: type) -> bool:
             if cls is not DeprecatedClass.deprecated_class:
                 # we should do the magic only if second `issubclass` argument
                 # is the deprecated class itself - subclasses of the
@@ -98,7 +101,7 @@ def create_deprecated_class(
             mro = getattr(sub, "__mro__", ())
             return any(c in {cls, new_class} for c in mro)
 
-        def __call__(cls, *args, **kwargs):
+        def __call__(cls, *args: Any, **kwargs: Any) -> Any:
             old = DeprecatedClass.deprecated_class
             if cls is old:
                 msg = instance_warn_message.format(
@@ -125,7 +128,7 @@ def create_deprecated_class(
     return deprecated_cls
 
 
-def _clspath(cls, forced=None):
+def _clspath(cls: type, forced: Optional[str] = None) -> str:
     if forced is not None:
         return forced
     return f"{cls.__module__}.{cls.__name__}"
@@ -134,7 +137,17 @@ def _clspath(cls, forced=None):
 DEPRECATION_RULES: List[Tuple[str, str]] = []
 
 
-def update_classpath(path):
+@overload
+def update_classpath(path: str) -> str:
+    ...
+
+
+@overload
+def update_classpath(path: Any) -> Any:
+    ...
+
+
+def update_classpath(path: Any) -> Any:
     """Update a deprecated path from an object with its new location"""
     for prefix, replacement in DEPRECATION_RULES:
         if isinstance(path, str) and path.startswith(prefix):
@@ -147,7 +160,7 @@ def update_classpath(path):
     return path
 
 
-def method_is_overridden(subclass, base_class, method_name):
+def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool:
     """
     Return True if a method named ``method_name`` of a ``base_class``
     is overridden in a ``subclass``.

From b8277f4cab7941989402a8339634a91dcd3500c4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 19:08:46 +0400
Subject: [PATCH 15/42] Add more typing to scrapy/utils/defer.py.

---
 scrapy/utils/defer.py | 57 ++++++++++++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 20 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d25ebbdf4..307707bf5 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -9,13 +9,16 @@ from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
+    AsyncIterator,
     Callable,
     Coroutine,
+    Dict,
     Generator,
     Iterable,
     Iterator,
     List,
     Optional,
+    Tuple,
     Union,
     cast,
 )
@@ -44,7 +47,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     return d
 
 
-def defer_succeed(result) -> Deferred:
+def defer_succeed(result: Any) -> Deferred:
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
 
@@ -58,7 +61,7 @@ def defer_succeed(result) -> Deferred:
     return d
 
 
-def defer_result(result) -> Deferred:
+def defer_result(result: Any) -> Deferred:
     if isinstance(result, Deferred):
         return result
     if isinstance(result, failure.Failure):
@@ -66,7 +69,7 @@ def defer_result(result) -> Deferred:
     return defer_succeed(result)
 
 
-def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
+def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred:
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
     """
@@ -84,7 +87,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
 
 
 def parallel(
-    iterable: Iterable, count: int, callable: Callable, *args, **named
+    iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any
 ) -> Deferred:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
@@ -146,14 +149,14 @@ class _AsyncCooperatorAdapter(Iterator):
         self,
         aiterable: AsyncIterable,
         callable: Callable,
-        *callable_args,
-        **callable_kwargs
+        *callable_args: Any,
+        **callable_kwargs: Any,
     ):
-        self.aiterator = aiterable.__aiter__()
-        self.callable = callable
-        self.callable_args = callable_args
-        self.callable_kwargs = callable_kwargs
-        self.finished = False
+        self.aiterator: AsyncIterator = aiterable.__aiter__()
+        self.callable: Callable = callable
+        self.callable_args: Tuple[Any, ...] = callable_args
+        self.callable_kwargs: Dict[str, Any] = callable_kwargs
+        self.finished: bool = False
         self.waiting_deferreds: List[Deferred] = []
         self.anext_deferred: Optional[Deferred] = None
 
@@ -201,7 +204,11 @@ class _AsyncCooperatorAdapter(Iterator):
 
 
 def parallel_async(
-    async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named
+    async_iterable: AsyncIterable,
+    count: int,
+    callable: Callable,
+    *args: Any,
+    **named: Any,
 ) -> Deferred:
     """Like parallel but for async iterators"""
     coop = Cooperator()
@@ -210,7 +217,9 @@ def parallel_async(
     return dl
 
 
-def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
+def process_chain(
+    callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any
+) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
     d: Deferred = Deferred()
     for x in callbacks:
@@ -220,7 +229,11 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
 
 
 def process_chain_both(
-    callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
+    callbacks: Iterable[Callable],
+    errbacks: Iterable[Callable],
+    input: Any,
+    *a: Any,
+    **kw: Any,
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d: Deferred = Deferred()
@@ -240,7 +253,9 @@ def process_chain_both(
     return d
 
 
-def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
+def process_parallel(
+    callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any
+) -> Deferred:
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
@@ -250,7 +265,9 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred
     return d
 
 
-def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
+def iter_errback(
+    iterable: Iterable, errback: Callable, *a: Any, **kw: Any
+) -> Generator:
     """Wraps an iterable calling an errback if an error is caught while
     iterating it.
     """
@@ -265,7 +282,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
 
 
 async def aiter_errback(
-    aiterable: AsyncIterable, errback: Callable, *a, **kw
+    aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any
 ) -> AsyncGenerator:
     """Wraps an async iterable calling an errback if an error is caught while
     iterating it. Similar to scrapy.utils.defer.iter_errback()
@@ -280,7 +297,7 @@ async def aiter_errback(
             errback(failure.Failure(), *a, **kw)
 
 
-def deferred_from_coro(o) -> Any:
+def deferred_from_coro(o: Any) -> Any:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, Deferred):
         return o
@@ -303,13 +320,13 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
     """
 
     @wraps(coro_f)
-    def f(*coro_args, **coro_kwargs):
+    def f(*coro_args: Any, **coro_kwargs: Any) -> Any:
         return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
 
     return f
 
 
-def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
+def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred:
     """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)

From c04b9ba19de85154c7bfec536f584f31456e44b3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 19:10:42 +0400
Subject: [PATCH 16/42] Add typing to scrapy/utils/template.py.

---
 scrapy/utils/template.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 1499aeb3d..6b22f3bfa 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -4,10 +4,10 @@ import re
 import string
 from os import PathLike
 from pathlib import Path
-from typing import Union
+from typing import Any, Union
 
 
-def render_templatefile(path: Union[str, PathLike], **kwargs):
+def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None:
     path_obj = Path(path)
     raw = path_obj.read_text("utf8")
 
@@ -24,7 +24,7 @@ def render_templatefile(path: Union[str, PathLike], **kwargs):
 CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")
 
 
-def string_camelcase(string):
+def string_camelcase(string: str) -> str:
     """Convert a word  to its CamelCase version and remove invalid chars
 
     >>> string_camelcase('lost-pound')

From 36507ddb7b55d6fc4bfdd19286d82057ef3fb5cf Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 19:16:57 +0400
Subject: [PATCH 17/42] Add typing to scrapy/utils/engine.py.

---
 scrapy/utils/engine.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 8e3ec2c37..a5f2a8c6e 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -2,9 +2,13 @@
 
 # used in global tests code
 from time import time  # noqa: F401
+from typing import TYPE_CHECKING, Any, List, Tuple
+
+if TYPE_CHECKING:
+    from scrapy.core.engine import ExecutionEngine
 
 
-def get_engine_status(engine):
+def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
     """Return a report of the current engine status"""
     tests = [
         "time()-engine.start_time",
@@ -23,7 +27,7 @@ def get_engine_status(engine):
         "engine.scraper.slot.needs_backout()",
     ]
 
-    checks = []
+    checks: List[Tuple[str, Any]] = []
     for test in tests:
         try:
             checks += [(test, eval(test))]
@@ -33,7 +37,7 @@ def get_engine_status(engine):
     return checks
 
 
-def format_engine_status(engine=None):
+def format_engine_status(engine: "ExecutionEngine") -> str:
     checks = get_engine_status(engine)
     s = "Execution engine status\n\n"
     for test, result in checks:
@@ -43,5 +47,5 @@ def format_engine_status(engine=None):
     return s
 
 
-def print_engine_status(engine):
+def print_engine_status(engine: "ExecutionEngine") -> None:
     print(format_engine_status(engine))

From f38cea9c8c5410e0553c666f090fb150a68ae591 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 19:22:57 +0400
Subject: [PATCH 18/42] Add typing to scrapy/utils/display.py.

---
 scrapy/utils/display.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 77c32b002..596cf89e4 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -6,17 +6,18 @@ import ctypes
 import platform
 import sys
 from pprint import pformat as pformat_
+from typing import Any
 
 from packaging.version import Version as parse_version
 
 
-def _enable_windows_terminal_processing():
+def _enable_windows_terminal_processing() -> bool:
     # https://stackoverflow.com/a/36760881
-    kernel32 = ctypes.windll.kernel32
+    kernel32 = ctypes.windll.kernel32  # type: ignore[attr-defined]
     return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
 
 
-def _tty_supports_color():
+def _tty_supports_color() -> bool:
     if sys.platform != "win32":
         return True
 
@@ -28,7 +29,7 @@ def _tty_supports_color():
     return _enable_windows_terminal_processing()
 
 
-def _colorize(text, colorize=True):
+def _colorize(text: str, colorize: bool = True) -> str:
     if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
@@ -42,9 +43,9 @@ def _colorize(text, colorize=True):
         return highlight(text, PythonLexer(), TerminalFormatter())
 
 
-def pformat(obj, *args, **kwargs):
+def pformat(obj: Any, *args: Any, **kwargs: Any) -> str:
     return _colorize(pformat_(obj), kwargs.pop("colorize", True))
 
 
-def pprint(obj, *args, **kwargs):
+def pprint(obj: Any, *args: Any, **kwargs: Any) -> None:
     print(pformat(obj, *args, **kwargs))

From 54fa04aa0a47314f121ff5a7627e7e47d4d2c013 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 20:32:34 +0400
Subject: [PATCH 19/42] Add typing to scrapy/utils/test.py, fix a FTP test.

---
 scrapy/crawler.py            | 18 +++++++------
 scrapy/spiderloader.py       | 11 +++++---
 scrapy/utils/test.py         | 49 ++++++++++++++++++++++++------------
 tests/test_pipeline_files.py | 41 +++++++++++++++---------------
 4 files changed, 71 insertions(+), 48 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 256f6e2c5..9631c73d6 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,11 +4,13 @@ import logging
 import pprint
 import signal
 import warnings
-from typing import TYPE_CHECKING, Optional, Type, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union
 
 from twisted.internet import defer
 from zope.interface.exceptions import DoesNotImplement
 
+from scrapy.spiderloader import SpiderLoader
+
 try:
     # zope >= 5.0 only supports MultipleInvalid
     from zope.interface.exceptions import MultipleInvalid
@@ -171,7 +173,7 @@ class CrawlerRunner:
     )
 
     @staticmethod
-    def _get_spider_loader(settings):
+    def _get_spider_loader(settings) -> SpiderLoader:
         """Get SpiderLoader instance from settings"""
         cls_path = settings.get("SPIDER_LOADER_CLASS")
         loader_cls = load_object(cls_path)
@@ -190,13 +192,13 @@ class CrawlerRunner:
             )
         return loader_cls.from_settings(settings.frozencopy())
 
-    def __init__(self, settings=None):
+    def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         self.settings = settings
         self.spider_loader = self._get_spider_loader(settings)
-        self._crawlers = set()
-        self._active = set()
+        self._crawlers: Set[Crawler] = set()
+        self._active: Set[defer.Deferred] = set()
         self.bootstrap_failed = False
 
     @property
@@ -252,7 +254,9 @@ class CrawlerRunner:
 
         return d.addBoth(_done)
 
-    def create_crawler(self, crawler_or_spidercls):
+    def create_crawler(
+        self, crawler_or_spidercls: Union[Type[Spider], str, Crawler]
+    ) -> Crawler:
         """
         Return a :class:`~scrapy.crawler.Crawler` object.
 
@@ -272,7 +276,7 @@ class CrawlerRunner:
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)
 
-    def _create_crawler(self, spidercls):
+    def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler:
         if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings)
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 02a451a2b..ea5a26e77 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,10 +1,13 @@
 import traceback
 import warnings
 from collections import defaultdict
+from typing import DefaultDict, Dict, List, Tuple, Type
 
 from zope.interface import implementer
 
+from scrapy import Spider
 from scrapy.interfaces import ISpiderLoader
+from scrapy.settings import BaseSettings
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.spider import iter_spider_classes
 
@@ -16,11 +19,11 @@ class SpiderLoader:
     in a Scrapy project.
     """
 
-    def __init__(self, settings):
+    def __init__(self, settings: BaseSettings):
         self.spider_modules = settings.getlist("SPIDER_MODULES")
         self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY")
-        self._spiders = {}
-        self._found = defaultdict(list)
+        self._spiders: Dict[str, Type[Spider]] = {}
+        self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list)
         self._load_all_spiders()
 
     def _check_name_duplicates(self):
@@ -68,7 +71,7 @@ class SpiderLoader:
     def from_settings(cls, settings):
         return cls(settings)
 
-    def load(self, spider_name):
+    def load(self, spider_name: str) -> Type[Spider]:
         """
         Return the Spider class for the given spider name. If the spider
         name is not found, raise a KeyError.
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 58576903a..97de8d25a 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,24 +7,30 @@ import os
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
-from unittest import mock
+from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type
+from unittest import TestCase, mock
 
+from twisted.internet.defer import Deferred
 from twisted.trial.unittest import SkipTest
 
+from scrapy import Spider
+from scrapy.crawler import Crawler
 from scrapy.utils.boto import is_botocore_available
 
 
-def assert_gcs_environ():
+def assert_gcs_environ() -> None:
     if "GCS_PROJECT_ID" not in os.environ:
         raise SkipTest("GCS_PROJECT_ID not found")
 
 
-def skip_if_no_boto():
+def skip_if_no_boto() -> None:
     if not is_botocore_available():
         raise SkipTest("missing botocore library")
 
 
-def get_gcs_content_and_delete(bucket, path):
+def get_gcs_content_and_delete(
+    bucket: Any, path: str
+) -> Tuple[bytes, List[Dict[str, str]], Any]:
     from google.cloud import storage
 
     client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
@@ -37,8 +43,13 @@ def get_gcs_content_and_delete(bucket, path):
 
 
 def get_ftp_content_and_delete(
-    path, host, port, username, password, use_active_mode=False
-):
+    path: str,
+    host: str,
+    port: int,
+    username: str,
+    password: str,
+    use_active_mode: bool = False,
+) -> bytes:
     from ftplib import FTP
 
     ftp = FTP()
@@ -46,19 +57,23 @@ def get_ftp_content_and_delete(
     ftp.login(username, password)
     if use_active_mode:
         ftp.set_pasv(False)
-    ftp_data = []
+    ftp_data: List[bytes] = []
 
-    def buffer_data(data):
+    def buffer_data(data: bytes) -> None:
         ftp_data.append(data)
 
     ftp.retrbinary(f"RETR {path}", buffer_data)
     dirname, filename = split(path)
     ftp.cwd(dirname)
     ftp.delete(filename)
-    return "".join(ftp_data)
+    return b"".join(ftp_data)
 
 
-def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
+def get_crawler(
+    spidercls: Optional[Type[Spider]] = None,
+    settings_dict: Optional[Dict[str, Any]] = None,
+    prevent_warnings: bool = True,
+) -> Crawler:
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
     priority.
@@ -82,7 +97,7 @@ def get_pythonpath() -> str:
     return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "")
 
 
-def get_testenv():
+def get_testenv() -> Dict[str, str]:
     """Return a OS environment dict suitable to fork processes that need to import
     this installation of Scrapy, instead of a system installed one.
     """
@@ -91,21 +106,23 @@ def get_testenv():
     return env
 
 
-def assert_samelines(testcase, text1, text2, msg=None):
+def assert_samelines(
+    testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None
+) -> None:
     """Asserts text1 and text2 have the same lines, ignoring differences in
     line endings between platforms
     """
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
 
-def get_from_asyncio_queue(value):
-    q = asyncio.Queue()
+def get_from_asyncio_queue(value: Any) -> Coroutine:
+    q: asyncio.Queue = asyncio.Queue()
     getter = q.get()
     q.put_nowait(value)
     return getter
 
 
-def mock_google_cloud_storage():
+def mock_google_cloud_storage() -> Tuple[Any, Any, Any]:
     """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
     classes and set their proper return values.
     """
@@ -122,7 +139,7 @@ def mock_google_cloud_storage():
     return (client_mock, bucket_mock, blob_mock)
 
 
-def get_web_client_agent_req(url):
+def get_web_client_agent_req(url: str) -> Deferred:
     from twisted.internet import reactor
     from twisted.web.client import Agent  # imports twisted.internet.reactor
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c80666586..859ad6f9c 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -32,6 +32,7 @@ from scrapy.utils.test import (
     get_gcs_content_and_delete,
     skip_if_no_boto,
 )
+from tests.mockserver import MockFTPServer
 
 from .test_pipeline_media import _mocked_download_func
 
@@ -639,31 +640,29 @@ class TestGCSFilesStore(unittest.TestCase):
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):
-        uri = os.environ.get("FTP_TEST_FILE_URI")
-        if not uri:
-            raise unittest.SkipTest("No FTP URI available for testing")
         data = b"TestFTPFilesStore: \xe2\x98\x83"
         buf = BytesIO(data)
         meta = {"foo": "bar"}
         path = "full/filename"
-        store = FTPFilesStore(uri)
-        empty_dict = yield store.stat_file(path, info=None)
-        self.assertEqual(empty_dict, {})
-        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
-        stat = yield store.stat_file(path, info=None)
-        self.assertIn("last_modified", stat)
-        self.assertIn("checksum", stat)
-        self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6")
-        path = f"{store.basedir}/{path}"
-        content = get_ftp_content_and_delete(
-            path,
-            store.host,
-            store.port,
-            store.username,
-            store.password,
-            store.USE_ACTIVE_MODE,
-        )
-        self.assertEqual(data.decode(), content)
+        with MockFTPServer() as ftp_server:
+            store = FTPFilesStore(ftp_server.url("/"))
+            empty_dict = yield store.stat_file(path, info=None)
+            self.assertEqual(empty_dict, {})
+            yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+            stat = yield store.stat_file(path, info=None)
+            self.assertIn("last_modified", stat)
+            self.assertIn("checksum", stat)
+            self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6")
+            path = f"{store.basedir}/{path}"
+            content = get_ftp_content_and_delete(
+                path,
+                store.host,
+                store.port,
+                store.username,
+                store.password,
+                store.USE_ACTIVE_MODE,
+            )
+        self.assertEqual(data, content)
 
 
 class ItemWithFiles(Item):

From 048812ba350d9f7fe6831d4c6f0d60d222b7f131 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 22:40:29 +0400
Subject: [PATCH 20/42] Bump types-* versions.

---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index af8f1f57a..a1e956bf4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -40,10 +40,10 @@ deps =
     mypy==1.2.0
     types-attrs==19.1.0
     types-lxml==2023.3.28
-    types-Pillow==9.4.0.19
-    types-Pygments==2.14.0.7
-    types-pyOpenSSL==23.1.0.1
-    types-setuptools==67.6.0.7
+    types-Pillow==9.5.0.2
+    types-Pygments==2.15.0.0
+    types-pyOpenSSL==23.1.0.2
+    types-setuptools==67.7.0.1
 commands =
     mypy --show-error-codes {posargs: scrapy tests}
 

From 0ec79e316619c1c98b0a1dd4fb0edea6e6de803d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 23:01:27 +0400
Subject: [PATCH 21/42] Fix compatibility with Python 3.8.

---
 scrapy/utils/deprecate.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index ab2719bb3..ea577c44a 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -2,7 +2,7 @@
 
 import inspect
 import warnings
-from typing import Any, List, Optional, Tuple, Type, overload
+from typing import Any, Dict, List, Optional, Tuple, Type, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -20,7 +20,7 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No
 def create_deprecated_class(
     name: str,
     new_class: type,
-    clsdict: Optional[dict[str, Any]] = None,
+    clsdict: Optional[Dict[str, Any]] = None,
     warn_category: Type[Warning] = ScrapyDeprecationWarning,
     warn_once: bool = True,
     old_class_path: Optional[str] = None,
@@ -59,14 +59,14 @@ def create_deprecated_class(
         warned_on_subclass: bool = False
 
         def __new__(
-            metacls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]
+            metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]
         ) -> type:
             cls = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
 
-        def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]):
+        def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]):
             meta = cls.__class__
             old = meta.deprecated_class
             if old in bases and not (warn_once and meta.warned_on_subclass):

From e03c6bb70a8915f997771472ef05efc0c3ad6bd6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 7 May 2023 23:03:35 +0400
Subject: [PATCH 22/42] Fix pylint issues.

---
 scrapy/crawler.py  | 3 +--
 scrapy/utils/gz.py | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 9631c73d6..69ff07bb7 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -9,8 +9,6 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union
 from twisted.internet import defer
 from zope.interface.exceptions import DoesNotImplement
 
-from scrapy.spiderloader import SpiderLoader
-
 try:
     # zope >= 5.0 only supports MultipleInvalid
     from zope.interface.exceptions import MultipleInvalid
@@ -27,6 +25,7 @@ from scrapy.interfaces import ISpiderLoader
 from scrapy.logformatter import LogFormatter
 from scrapy.settings import Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
+from scrapy.spiderloader import SpiderLoader
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.log import (
     LogCounterHandler,
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 98ca510ed..c0eb77e07 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -24,8 +24,7 @@ def gunzip(data: bytes) -> bytes:
             # some pages are quite small so output_list is empty
             if output_list:
                 break
-            else:
-                raise
+            raise
     return b"".join(output_list)
 
 

From 6998e1c905ef6e5fa737b32ac0b6e7f1b7701c14 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 10 May 2023 14:21:18 +0400
Subject: [PATCH 23/42] Fix typing-related issued on Python < 3.9.

---
 scrapy/utils/datatypes.py | 7 +++----
 scrapy/utils/misc.py      | 4 +---
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 599b201ea..0f6bdc5ab 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -5,10 +5,9 @@ Python Standard Library.
 This module must not depend on any module outside the Standard Library.
 """
 
-import collections
 import weakref
 from collections.abc import Mapping
-from typing import Any, Optional, Sequence, TypeVar
+from typing import Any, Optional, OrderedDict, Sequence, TypeVar
 
 _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
@@ -68,7 +67,7 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class LocalCache(collections.OrderedDict[_KT, _VT]):
+class LocalCache(OrderedDict[_KT, _VT]):
     """Dictionary with a finite number of keys.
 
     Older items expires first.
@@ -85,7 +84,7 @@ class LocalCache(collections.OrderedDict[_KT, _VT]):
         super().__setitem__(key, value)
 
 
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]):
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
     """
     A weakref.WeakKeyDictionary implementation that uses LocalCache as its
     underlying data structure, making it ordered and capable of being size-limited.
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index defc8663d..70187ba74 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -232,9 +232,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
         yield node
 
 
-_generator_callbacks_cache: LocalWeakReferencedCache[
-    Callable, bool
-] = LocalWeakReferencedCache(limit=128)
+_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
 def is_generator_with_return_value(callable: Callable) -> bool:

From 49839d6071832aab23093c34fa6ceb961fcdf9d0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 4 Jun 2023 19:59:58 +0400
Subject: [PATCH 24/42] Don't rely on get_testenv() for running mockserver.

---
 tests/mockserver.py         | 19 +++++++++++++++----
 tests/test_crawler.py       | 11 ++++++++---
 tests/test_proxy_connect.py |  3 ---
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index eb4f8334e..647b0682e 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,11 +1,13 @@
 import argparse
 import json
+import os
 import random
 import sys
 from pathlib import Path
 from shutil import rmtree
 from subprocess import PIPE, Popen
 from tempfile import mkdtemp
+from typing import Dict
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
@@ -20,7 +22,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.test import get_testenv
 
 
 def getarg(request, name, default=None, type=None):
@@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None):
     return default
 
 
+def get_mockserver_env() -> Dict[str, str]:
+    """Return a OS environment dict suitable to run mockserver processes."""
+
+    tests_path = Path(__file__).parent.parent
+    pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "")
+    env = os.environ.copy()
+    env["PYTHONPATH"] = pythonpath
+    return env
+
+
 # most of the following resources are copied from twisted.web.test.test_webclient
 class ForeverTakingResource(resource.Resource):
     """
@@ -264,7 +275,7 @@ class MockServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         http_address = self.proc.stdout.readline().strip().decode("ascii")
         https_address = self.proc.stdout.readline().strip().decode("ascii")
@@ -308,7 +319,7 @@ class MockDNSServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         self.host = "127.0.0.1"
         self.port = int(
@@ -331,7 +342,7 @@ class MockFTPServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)],
             stderr=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         for line in self.proc.stderr:
             if b"starting FTP server" in line:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 706bfbaa9..ecb9c9b62 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,5 @@
 import logging
+import os
 import platform
 import subprocess
 import sys
@@ -23,8 +24,8 @@ from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.misc import load_object
 from scrapy.utils.spider import DefaultSpider
-from scrapy.utils.test import get_crawler, get_testenv
-from tests.mockserver import MockServer
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer, get_mockserver_env
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -289,12 +290,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
 class ScriptRunnerMixin:
     script_dir: Path
+    cwd = os.getcwd()
 
     def run_script(self, script_name: str, *script_args):
         script_path = self.script_dir / script_name
         args = [sys.executable, str(script_path)] + list(script_args)
         p = subprocess.Popen(
-            args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            args,
+            env=get_mockserver_env(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         stdout, stderr = p.communicate()
         return stderr.decode("utf-8")
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index c05f4da91..dc0a82086 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -21,8 +21,6 @@ class MitmProxy:
     auth_pass = "scrapy"
 
     def start(self):
-        from scrapy.utils.test import get_testenv
-
         script = """
 import sys
 from mitmproxy.tools.main import mitmdump
@@ -46,7 +44,6 @@ sys.exit(mitmdump())
                 "--ssl-insecure",
             ],
             stdout=PIPE,
-            env=get_testenv(),
         )
         line = self.proc.stdout.readline().decode("utf-8")
         host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)

From 493ea435384d4b5891129cbcf61d07b09a00ce7f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 12 Jun 2023 21:50:29 +0400
Subject: [PATCH 25/42] Improve finding tests.test_cmdline.settings.

---
 tests/test_cmdline/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 15833cd19..25ded143c 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,4 +1,5 @@
 import json
+import os
 import pstats
 import shutil
 import sys
@@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv
 class CmdlineTest(unittest.TestCase):
     def setUp(self):
         self.env = get_testenv()
+        tests_path = Path(__file__).parent.parent
+        self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
         self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
 
     def _execute(self, *new_args, **kwargs):

From 7cfdca8f9b1b1f16aacae958f7a5c8824df056e7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 14 Jun 2023 18:23:29 +0400
Subject: [PATCH 26/42] Actually run test_set_asyncio_event_loop().

---
 tests/test_utils_asyncio.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 01d0ee043..65e352053 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -4,6 +4,7 @@ from unittest import TestCase
 
 from pytest import mark
 
+from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.reactor import (
     install_reactor,
     is_asyncio_reactor_installed,
@@ -29,6 +30,8 @@ class AsyncioTest(TestCase):
 
         assert original_reactor == reactor
 
+    @mark.only_asyncio()
+    @deferred_f_from_coro_f
     async def test_set_asyncio_event_loop(self):
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
-        assert set_asyncio_event_loop() is asyncio.get_running_loop()
+        assert set_asyncio_event_loop(None) is asyncio.get_running_loop()

From 777a6ea4128cf00d53fa71dc48385bf036cb64fc Mon Sep 17 00:00:00 2001
From: Serhii A <aserhii@protonmail.com>
Date: Fri, 16 Jun 2023 16:46:06 +0300
Subject: [PATCH 27/42] Make the retry middleware exception list configurable
 (#5929)

---
 docs/topics/downloader-middleware.rst    | 32 ++++++++++++
 scrapy/downloadermiddlewares/retry.py    | 63 ++++++++++++------------
 scrapy/settings/default_settings.py      | 15 ++++++
 tests/test_downloadermiddleware_retry.py | 39 +++++++++++++--
 4 files changed, 113 insertions(+), 36 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 7665a901a..a8e5b23bf 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -915,6 +915,7 @@ settings (see the settings documentation for more info):
 * :setting:`RETRY_ENABLED`
 * :setting:`RETRY_TIMES`
 * :setting:`RETRY_HTTP_CODES`
+* :setting:`RETRY_EXCEPTIONS`
 
 .. reqmeta:: dont_retry
 
@@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
 it is a common code used to indicate server overload. It is not included by
 default because HTTP specs say so.
 
+.. setting:: RETRY_EXCEPTIONS
+
+RETRY_EXCEPTIONS
+^^^^^^^^^^^^^^^^
+
+Default::
+
+    [
+        'twisted.internet.defer.TimeoutError',
+        'twisted.internet.error.TimeoutError',
+        'twisted.internet.error.DNSLookupError',
+        'twisted.internet.error.ConnectionRefusedError',
+        'twisted.internet.error.ConnectionDone',
+        'twisted.internet.error.ConnectError',
+        'twisted.internet.error.ConnectionLost',
+        'twisted.internet.error.TCPTimedOutError',
+        'twisted.web.client.ResponseFailed',
+        IOError,
+        'scrapy.core.downloader.handlers.http11.TunnelError',
+    ]
+
+List of exceptions to retry.
+
+Each list entry may be an exception type or its import path as a string.
+
+An exception will not be caught when the exception type is not in
+:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
+has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
+exception propagation, see
+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
+
 .. setting:: RETRY_PRIORITY_ADJUST
 
 RETRY_PRIORITY_ADJUST
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 081642a4b..50cbc3111 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+import warnings
 from logging import Logger, getLogger
 from typing import Optional, Union
 
-from twisted.internet import defer
-from twisted.internet.error import (
-    ConnectError,
-    ConnectionDone,
-    ConnectionLost,
-    ConnectionRefusedError,
-    DNSLookupError,
-    TCPTimedOutError,
-    TimeoutError,
-)
-from twisted.web.client import ResponseFailed
-
-from scrapy.core.downloader.handlers.http11 import TunnelError
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http.request import Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
 
 retry_logger = getLogger(__name__)
 
 
+class BackwardsCompatibilityMetaclass(type):
+    @property
+    def EXCEPTIONS_TO_RETRY(cls):
+        warnings.warn(
+            "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
+            "Use the RETRY_EXCEPTIONS setting instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return tuple(
+            load_object(x) if isinstance(x, str) else x
+            for x in Settings().getlist("RETRY_EXCEPTIONS")
+        )
+
+
 def get_retry_request(
     request: Request,
     *,
@@ -121,23 +126,7 @@ def get_retry_request(
     return None
 
 
-class RetryMiddleware:
-    # IOError is raised by the HttpCompression middleware when trying to
-    # decompress an empty response
-    EXCEPTIONS_TO_RETRY = (
-        defer.TimeoutError,
-        TimeoutError,
-        DNSLookupError,
-        ConnectionRefusedError,
-        ConnectionDone,
-        ConnectError,
-        ConnectionLost,
-        TCPTimedOutError,
-        ResponseFailed,
-        IOError,
-        TunnelError,
-    )
-
+class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
     def __init__(self, settings):
         if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
@@ -147,6 +136,16 @@ class RetryMiddleware:
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
+        if not hasattr(
+            self, "EXCEPTIONS_TO_RETRY"
+        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+            self.exceptions_to_retry = tuple(
+                load_object(x) if isinstance(x, str) else x
+                for x in settings.getlist("RETRY_EXCEPTIONS")
+            )
+        else:
+            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
@@ -160,7 +159,7 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
+        if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
             "dont_retry", False
         ):
             return self._retry(request, exception, spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 260ec1701..89837b4ab 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -258,6 +258,21 @@ RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
+RETRY_EXCEPTIONS = [
+    "twisted.internet.defer.TimeoutError",
+    "twisted.internet.error.TimeoutError",
+    "twisted.internet.error.DNSLookupError",
+    "twisted.internet.error.ConnectionRefusedError",
+    "twisted.internet.error.ConnectionDone",
+    "twisted.internet.error.ConnectError",
+    "twisted.internet.error.ConnectionLost",
+    "twisted.internet.error.TCPTimedOutError",
+    "twisted.web.client.ResponseFailed",
+    # IOError is raised by the HttpCompression middleware when trying to
+    # decompress an empty response
+    IOError,
+    "scrapy.core.downloader.handlers.http11.TunnelError",
+]
 
 ROBOTSTXT_OBEY = False
 ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 63bd61848..97ae1e29a 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,5 +1,6 @@
 import logging
 import unittest
+import warnings
 
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
+from scrapy.settings.default_settings import RETRY_EXCEPTIONS
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
@@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase):
             == 2
         )
 
-    def _test_retry_exception(self, req, exception):
+    def test_exception_to_retry_added(self):
+        exc = ValueError
+        settings_dict = {
+            "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc],
+        }
+        crawler = get_crawler(Spider, settings_dict=settings_dict)
+        mw = RetryMiddleware.from_crawler(crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        self._test_retry_exception(req, exc("foo"), mw)
+
+    def test_exception_to_retry_customMiddleware(self):
+        exc = ValueError
+
+        with warnings.catch_warnings(record=True) as warns:
+
+            class MyRetryMiddleware(RetryMiddleware):
+                EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
+
+            self.assertEqual(len(warns), 1)
+
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+
+    def _test_retry_exception(self, req, exception, mw=None):
+        if mw is None:
+            mw = self.mw
+
         # first retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
         # second retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 2)
 
         # discard it
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         self.assertEqual(req, None)
 
 

From 2122278d4bb7177a550bc0b04dd96d1fc1fad1a0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 18 Jun 2023 18:37:50 +0400
Subject: [PATCH 28/42] Drop Python 3.7 support.

---
 .github/workflows/tests-macos.yml   |  2 +-
 .github/workflows/tests-ubuntu.yml  | 13 +++++--------
 .github/workflows/tests-windows.yml |  5 +----
 README.rst                          |  2 +-
 docs/contributing.rst               | 12 ++++++------
 docs/intro/install.rst              |  2 +-
 scrapy/__init__.py                  |  4 ++--
 setup.py                            |  3 +--
 tox.ini                             |  8 +++-----
 9 files changed, 21 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 174d245ca..3044a1af3 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 96b26a1f8..39e3b0af7 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -8,9 +8,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.8
-          env:
-            TOXENV: py
         - python-version: 3.9
           env:
             TOXENV: py
@@ -28,19 +25,19 @@ jobs:
             TOXENV: pypy3
 
         # pinned deps
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: asyncio-pinned
-        - python-version: pypy3.7
+        - python-version: pypy3.8
           env:
             TOXENV: pypy3-pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: extra-deps-pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: botocore-pinned
 
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index f60c48841..5bcf74d5e 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -8,12 +8,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.7
-          env:
-            TOXENV: windows-pinned
         - python-version: 3.8
           env:
-            TOXENV: py
+            TOXENV: windows-pinned
         - python-version: 3.9
           env:
             TOXENV: py
diff --git a/README.rst b/README.rst
index 970bf2c35..1918850d6 100644
--- a/README.rst
+++ b/README.rst
@@ -58,7 +58,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.7+
+* Python 3.8+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/contributing.rst b/docs/contributing.rst
index eef92e148..2b3249601 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
 To run the tests on a specific :doc:`tox <tox:index>` environment, use
 ``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.7 use::
+the tests with Python 3.10 use::
 
-    tox -e py37
+    tox -e py310
 
 You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
-    tox -e py37,py38 -p auto
+    tox -e py39,py310 -p auto
 
 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
     tox -- scrapy tests -x  # stop after first failure
 
 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
 
-    tox -e py37 -- scrapy tests -n auto
+    tox -e py310 -- scrapy tests -n auto
 
 To see coverage report install :doc:`coverage <coverage:index>`
 (``pip install coverage``) and run:
@@ -322,4 +322,4 @@ And their unit-tests are in::
 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
 .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
-.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
\ No newline at end of file
+.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 2c2079f68..c90c1d2bf 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,7 +9,7 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.7+, either the CPython implementation (default) or
+Scrapy requires Python 3.8+, either the CPython implementation (default) or
 the PyPy implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index a757a9290..cc0e539c4 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 7):
-    print(f"Scrapy {__version__} requires Python 3.7+")
+if sys.version_info < (3, 8):
+    print(f"Scrapy {__version__} requires Python 3.8+")
     sys.exit(1)
 
 
diff --git a/setup.py b/setup.py
index c6bcf2439..f1cd4c5e2 100644
--- a/setup.py
+++ b/setup.py
@@ -80,7 +80,6 @@ setup(
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
@@ -91,7 +90,7 @@ setup(
         "Topic :: Software Development :: Libraries :: Application Frameworks",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tox.ini b/tox.ini
index 5f8bf85f2..d5b6118f5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,8 +16,6 @@ deps =
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
     mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
-    # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
-    markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -96,7 +94,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
 
 [testenv:pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     PyDispatcher==2.0.5
@@ -129,7 +127,7 @@ deps =
     Twisted[http2]
 
 [testenv:extra-deps-pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     boto3==1.20.0
@@ -211,7 +209,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
 
 [testenv:botocore-pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     botocore==1.4.87

From 1b2c9a3e0ae9766623a06ce7e739a3dfda399159 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 18 Jun 2023 18:38:56 +0400
Subject: [PATCH 29/42] Bump isort and flake8 versions.

---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index faf8808f2..31e9ed1ad 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ repos:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4  # 6.0.0 drops Python 3.7 support
+  rev: 6.0.0
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
@@ -13,7 +13,7 @@ repos:
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.11.5  # 5.12 drops Python 3.7 support
+  rev: 5.12.0
   hooks:
   - id: isort
 - repo: https://github.com/adamchainz/blacken-docs

From 075b89eab5e201246a5bddc4a6ce9c7921de082b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 18 Jun 2023 19:08:41 +0400
Subject: [PATCH 30/42] Bump lxml and cryptography to versions with 3.8 wheels
 available.

---
 setup.py | 4 ++--
 tox.ini  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index f1cd4c5e2..ccfe20ae5 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     "Twisted>=18.9.0",
-    "cryptography>=3.4.6",
+    "cryptography>=36.0.0",
     "cssselect>=0.9.1",
     "itemloaders>=1.0.1",
     "parsel>=1.5.0",
@@ -34,7 +34,7 @@ install_requires = [
     "setuptools",
     "packaging",
     "tldextract",
-    "lxml>=4.3.0",
+    "lxml>=4.4.1",
 ]
 extras_require = {}
 cpython_dependencies = [
diff --git a/tox.ini b/tox.ini
index d5b6118f5..ec3a59366 100644
--- a/tox.ini
+++ b/tox.ini
@@ -69,7 +69,7 @@ commands =
 
 [pinned]
 deps =
-    cryptography==3.4.6
+    cryptography==36.0.0
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
@@ -81,7 +81,7 @@ deps =
     Twisted[http2]==18.9.0
     w3lib==1.17.0
     zope.interface==5.1.0
-    lxml==4.3.0
+    lxml==4.4.1
     -rtests/requirements.txt
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies

From 0097b4c0bb4de6e651e8b9d064aae140e11698d5 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 16:40:38 -0300
Subject: [PATCH 31/42] cleanup: Remove `pkg_resources` usage

---
 docs/topics/components.rst | 2 +-
 scrapy/cmdline.py          | 5 ++---
 setup.py                   | 2 +-
 tests/test_crawler.py      | 2 +-
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index 1ed55f000..478dd9647 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use
 
 .. code-block:: python
 
-    from pkg_resources import parse_version
+    from packaging.version import parse as parse_version
 
     import scrapy
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 730e55350..cfa771104 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -3,8 +3,7 @@ import cProfile
 import inspect
 import os
 import sys
-
-import pkg_resources
+from importlib.metadata import entry_points
 
 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@@ -49,7 +48,7 @@ def _get_commands_from_module(module, inproject):
 
 def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
-    for entry_point in pkg_resources.iter_entry_points(group):
+    for entry_point in entry_points(group):
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()
diff --git a/setup.py b/setup.py
index ccfe20ae5..f918db09e 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ecb9c9b62..d54a2cb7e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -6,7 +6,7 @@ import sys
 import warnings
 from pathlib import Path
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted import version as twisted_version
 from twisted.internet import defer

From 6afb31b82b5a0a5d2f37962c250fbf34c21d8580 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 16:48:07 -0300
Subject: [PATCH 32/42] chore: Add `packaging` to tests deps

---
 tests/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index 618949795..72350b216 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -14,3 +14,4 @@ brotli  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
+packaging
\ No newline at end of file

From 6e1af20ac4dd537a4643df5c022f948cf07d05ec Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 17:00:01 -0300
Subject: [PATCH 33/42] fix: add `build-system`

---
 tox.ini | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tox.ini b/tox.ini
index ec3a59366..79d692599 100644
--- a/tox.ini
+++ b/tox.ini
@@ -218,3 +218,6 @@ setenv =
     {[pinned]setenv}
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[build-system]
+build-backend = 'setuptools.build_meta'
\ No newline at end of file

From a93a63c208af1d13d5ea84623d160337c7fec6c5 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 17:05:49 -0300
Subject: [PATCH 34/42] fix: move import to inside function

---
 setup.py               | 3 ++-
 tests/requirements.txt | 1 -
 tox.ini                | 3 ---
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index f918db09e..dfe5b80ec 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,5 @@
 from pathlib import Path
 
-from packaging.version import parse as parse_version
 from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
@@ -15,6 +14,8 @@ def has_environment_marker_platform_impl_support():
     it is 18.5, see:
     https://setuptools.readthedocs.io/en/latest/history.html#id235
     """
+    from packaging.version import parse as parse_version
+
     return parse_version(setuptools_version) >= parse_version("18.5")
 
 
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 72350b216..618949795 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -14,4 +14,3 @@ brotli  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
-packaging
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 79d692599..ec3a59366 100644
--- a/tox.ini
+++ b/tox.ini
@@ -218,6 +218,3 @@ setenv =
     {[pinned]setenv}
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
-
-[build-system]
-build-backend = 'setuptools.build_meta'
\ No newline at end of file

From 0b1da44a05cc64970aa11ccc4d7a4a3bec143443 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 17:14:21 -0300
Subject: [PATCH 35/42] chore: Remove deprecated code

---
 setup.py | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/setup.py b/setup.py
index dfe5b80ec..1f214571b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,24 +1,10 @@
 from pathlib import Path
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
 version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
 
 
-def has_environment_marker_platform_impl_support():
-    """Code extracted from 'pytest/setup.py'
-    https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31
-
-    The first known release to support environment marker with range operators
-    it is 18.5, see:
-    https://setuptools.readthedocs.io/en/latest/history.html#id235
-    """
-    from packaging.version import parse as parse_version
-
-    return parse_version(setuptools_version) >= parse_version("18.5")
-
-
 install_requires = [
     "Twisted>=18.9.0",
     "cryptography>=36.0.0",
@@ -37,19 +23,10 @@ install_requires = [
     "tldextract",
     "lxml>=4.4.1",
 ]
-extras_require = {}
-cpython_dependencies = [
-    "PyDispatcher>=2.0.5",
-]
-if has_environment_marker_platform_impl_support():
-    extras_require[
-        ':platform_python_implementation == "CPython"'
-    ] = cpython_dependencies
-    extras_require[':platform_python_implementation == "PyPy"'] = [
-        "PyPyDispatcher>=2.1.0",
-    ]
-else:
-    install_requires.extend(cpython_dependencies)
+extras_require = {
+    ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],
+    ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],
+}
 
 
 setup(

From 82cf00bbc931723320c9aca3cf0305372027d0a0 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 18:27:03 -0300
Subject: [PATCH 36/42] fix: default value

---
 scrapy/cmdline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index cfa771104..efc9b36ea 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -48,7 +48,7 @@ def _get_commands_from_module(module, inproject):
 
 def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
-    for entry_point in entry_points(group):
+    for entry_point in entry_points().get(group, {}):
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()

From ee215a29704adbc61a40baeca1d27f6179a1735e Mon Sep 17 00:00:00 2001
From: Aaron Smith <60046611+medic-code@users.noreply.github.com>
Date: Wed, 21 Jun 2023 19:05:39 +0100
Subject: [PATCH 37/42] Change redirect text from Response.request docs (#5937)

---
 docs/topics/request-response.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 407df32d2..41df51589 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1103,9 +1103,10 @@ Response objects
         through all :ref:`Downloader Middlewares <topics-downloader-middleware>`.
         In particular, this means that:
 
-        - HTTP redirections will cause the original request (to the URL before
-          redirection) to be assigned to the redirected response (with the final
-          URL after redirection).
+        - HTTP redirections will create a new request from the request before
+          redirection. It has the majority of the same metadata and original
+          request attributes and gets assigned to the redirected response
+          instead of the propagation of the original request.
 
         - Response.request.url doesn't always equal Response.url
 

From 5360ba34bc345667f77a4d4256f15fd648e42e18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Wed, 21 Jun 2023 11:08:53 -0700
Subject: [PATCH 38/42] IOError and other cleanup (#4716)

---
 docs/utils/linkfix.py                         | 2 +-
 scrapy/downloadermiddlewares/decompression.py | 4 ++--
 scrapy/downloadermiddlewares/httpcache.py     | 2 +-
 scrapy/settings/default_settings.py           | 4 ++--
 scrapy/utils/gz.py                            | 2 +-
 scrapy/utils/python.py                        | 2 +-
 tests/test_downloader_handlers.py             | 2 +-
 tests/test_downloadermiddleware.py            | 4 ++--
 tests/test_mail.py                            | 2 --
 tests/test_robotstxt_interface.py             | 1 -
 tests/test_utils_gz.py                        | 2 +-
 tests/test_utils_iterators.py                 | 4 ++--
 12 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 1f270837c..c17b9d511 100644
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -30,7 +30,7 @@ def main():
     try:
         with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
             output_lines = out.readlines()
-    except IOError:
+    except OSError:
         print("linkcheck output not found; please run linkcheck first.")
         sys.exit(1)
 
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 5839dc243..3b8702419 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -63,7 +63,7 @@ class DecompressionMiddleware:
         archive = BytesIO(response.body)
         try:
             body = gzip.GzipFile(fileobj=archive).read()
-        except IOError:
+        except OSError:
             return
 
         respcls = responsetypes.from_args(body=body)
@@ -72,7 +72,7 @@ class DecompressionMiddleware:
     def _is_bzip2(self, response):
         try:
             body = bz2.decompress(response.body)
-        except IOError:
+        except OSError:
             return
 
         respcls = responsetypes.from_args(body=body)
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index b9316c43a..ac87d4a4e 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -37,7 +37,7 @@ class HttpCacheMiddleware:
         ConnectionLost,
         TCPTimedOutError,
         ResponseFailed,
-        IOError,
+        OSError,
     )
 
     def __init__(self, settings: Settings, stats: StatsCollector) -> None:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 89837b4ab..a4cb555bd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -268,9 +268,9 @@ RETRY_EXCEPTIONS = [
     "twisted.internet.error.ConnectionLost",
     "twisted.internet.error.TCPTimedOutError",
     "twisted.web.client.ResponseFailed",
-    # IOError is raised by the HttpCompression middleware when trying to
+    # OSError is raised by the HttpCompression middleware when trying to
     # decompress an empty response
-    IOError,
+    OSError,
     "scrapy.core.downloader.handlers.http11.TunnelError",
 ]
 
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index e5df34d2e..77e0197d8 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -15,7 +15,7 @@ def gunzip(data):
         try:
             chunk = f.read1(8196)
             output_list.append(chunk)
-        except (IOError, EOFError, struct.error):
+        except (OSError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
             # some pages are quite small so output_list is empty and f.extrabuf
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 27816c0df..bb5dbebbc 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -291,7 +291,7 @@ def without_none_values(iterable):
     try:
         return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
-        return type(iterable)((v for v in iterable if v is not None))
+        return type(iterable)(v for v in iterable if v is not None)
 
 
 def global_object_name(obj):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index fd4176e2f..9731b62c4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase):
     def test_non_existent(self):
         request = Request(f"file://{self.mktemp()}")
         d = self.download_request(request, Spider("foo"))
-        return self.assertFailure(d, IOError)
+        return self.assertFailure(d, OSError)
 
 
 class ContentLengthHeaderResource(resource.Resource):
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 060cfe08b..062e8a8b4 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase):
         In particular when some website returns a 30x response with header
         'Content-Encoding: gzip' giving as result the error below:
 
-            exceptions.IOError: Not a gzipped file
+            BadGzipFile: Not a gzipped file (...)
 
         """
         req = Request("http://example.com")
@@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase):
                 "Location": "http://example.com/login",
             },
         )
-        self.assertRaises(IOError, self._download, request=req, response=resp)
+        self.assertRaises(OSError, self._download, request=req, response=resp)
 
 
 class ResponseFromProcessRequestTest(ManagerTestCase):
diff --git a/tests/test_mail.py b/tests/test_mail.py
index bc7298e9d..504c78486 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -1,5 +1,3 @@
-# coding=utf-8
-
 import unittest
 from email.charset import Charset
 from io import BytesIO
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 8d87a322a..d7a923085 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,4 +1,3 @@
-# coding=utf-8
 from twisted.trial import unittest
 
 
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 6b2a458bc..7b7a25db8 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_no_gzip_file_raises(self):
         self.assertRaises(
-            IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
+            OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
         )
 
     def test_gunzip_truncated_short(self):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index faf7d2709..3598fa0bb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
-            self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
+            self.assertTrue(all(isinstance(k, str) for k in result_row.keys()))
+            self.assertTrue(all(isinstance(v, str) for v in result_row.values()))
 
     def test_csviter_delimiter(self):
         body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")

From 04ee3303e4487270a433f5c3a087bda9a87d7008 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Wed, 21 Jun 2023 22:04:06 -0700
Subject: [PATCH 39/42] Adding support for Windows of absolute pathlib.Path
 objects in FeedExporter (#5939)

---
 docs/topics/feed-exports.rst    |  4 ++--
 scrapy/extensions/feedexport.py |  7 +++++--
 tests/test_feedexport.py        | 25 +++++++++++++++++++++++++
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index b31dc069e..aba47d998 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -156,8 +156,8 @@ The feeds are stored in the local filesystem.
 -   Required external libraries: none
 
 Note that for the local filesystem storage (only) you can omit the scheme if
-you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
-systems though.
+you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
+Alternatively you can also use a :class:`pathlib.Path` object.
 
 .. _topics-feed-storage-ftp:
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 39934cbf3..1cdc78f59 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -382,7 +382,9 @@ class FeedExporter:
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-            uri = str(self.settings["FEED_URI"])  # handle pathlib.Path objects
+            uri = self.settings["FEED_URI"]
+            # handle pathlib.Path objects
+            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
             feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
@@ -392,7 +394,8 @@ class FeedExporter:
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
         for uri, feed_options in self.settings.getdict("FEEDS").items():
-            uri = str(uri)  # handle pathlib.Path objects
+            # handle pathlib.Path objects
+            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
             )
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 62a5697cd..8df86dbd7 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase):
         with self.assertRaises(NotConfigured):
             FeedExporter.from_crawler(crawler)
 
+    def test_absolute_pathlib_as_uri(self):
+        with tempfile.NamedTemporaryFile(suffix="json") as tmp:
+            settings = {
+                "FEEDS": {
+                    Path(tmp.name).resolve(): {
+                        "format": "json",
+                    },
+                },
+            }
+            crawler = get_crawler(settings_dict=settings)
+            exporter = FeedExporter.from_crawler(crawler)
+            self.assertIsInstance(exporter, FeedExporter)
+
+    def test_relative_pathlib_as_uri(self):
+        settings = {
+            "FEEDS": {
+                Path("./items.json"): {
+                    "format": "json",
+                },
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        exporter = FeedExporter.from_crawler(crawler)
+        self.assertIsInstance(exporter, FeedExporter)
+
 
 class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
     def __init__(self, uri):

From 080b9bd0b8f65dcf09ea8ad94505fec6c77f820c Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 22 Jun 2023 23:58:03 -0300
Subject: [PATCH 40/42] chore: Implement `pop` method on `BaseSettings` class

---
 scrapy/settings/__init__.py     | 14 ++++++++++++++
 tests/test_settings/__init__.py | 13 +++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index a3b849f7b..57fe1d17a 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -75,6 +75,8 @@ class BaseSettings(MutableMapping):
     highest priority will be retrieved.
     """
 
+    __default = object()
+
     def __init__(self, values=None, priority="project"):
         self.frozen = False
         self.attributes = {}
@@ -445,6 +447,18 @@ class BaseSettings(MutableMapping):
         else:
             p.text(pformat(self.copy_to_dict()))
 
+    def pop(self, name, default=__default):
+        try:
+            value = self.attributes[name]
+        except KeyError:
+            if default is self.__default:
+                raise
+
+            return SettingsAttribute(default, get_settings_priority("project"))
+        else:
+            del self.attributes[name]
+            return value
+
 
 class Settings(BaseSettings):
     """
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 4a577cd8c..0e2f4aa98 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -451,6 +451,19 @@ class SettingsTest(unittest.TestCase):
         self.assertIsInstance(myhandler_instance, FileDownloadHandler)
         self.assertTrue(hasattr(myhandler_instance, "download_request"))
 
+    def test_pop_item_with_default_value(self):
+        settings = Settings()
+
+        with self.assertRaises(KeyError):
+            settings.pop("DUMMY_CONFIG")
+
+        dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value")
+
+        self.assertEqual(
+            repr(dummy_config), "<SettingsAttribute value='dummy_value' priority=20>"
+        )
+        self.assertEqual(dummy_config.value, "dummy_value")
+
 
 if __name__ == "__main__":
     unittest.main()

From 876feaf339e181c9c5a6b9a5f8ffedc03a9ed3d2 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 23 Jun 2023 00:14:31 -0300
Subject: [PATCH 41/42] chore: Use dunder to delete item instead of del keyword
 to handle immutable settings

---
 scrapy/settings/__init__.py     |  2 +-
 tests/test_settings/__init__.py | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 57fe1d17a..8b3bdbabe 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -456,7 +456,7 @@ class BaseSettings(MutableMapping):
 
             return SettingsAttribute(default, get_settings_priority("project"))
         else:
-            del self.attributes[name]
+            self.__delitem__(name)
             return value
 
 
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 0e2f4aa98..125b1d96f 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -464,6 +464,22 @@ class SettingsTest(unittest.TestCase):
         )
         self.assertEqual(dummy_config.value, "dummy_value")
 
+    def test_pop_item_with_frozen_settings(self):
+        settings = Settings(
+            {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
+        )
+
+        self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value")
+
+        settings.freeze()
+
+        with self.assertRaises(TypeError) as error:
+            settings.pop("OTHER_DUMMY_CONFIG")
+
+        self.assertEqual(
+            str(error.exception), "Trying to modify an immutable Settings object"
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

From a3f8912d69eacdd2208617e6afb418e4e1847e36 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 23 Jun 2023 00:15:32 -0300
Subject: [PATCH 42/42] chore: Rename test

---
 tests/test_settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 125b1d96f..bb6dc67fa 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -464,7 +464,7 @@ class SettingsTest(unittest.TestCase):
         )
         self.assertEqual(dummy_config.value, "dummy_value")
 
-    def test_pop_item_with_frozen_settings(self):
+    def test_pop_item_with_immutable_settings(self):
         settings = Settings(
             {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
         )