From 52d0df5f989903d46e6de4878e1d6a0e87a2c803 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 12 May 2021 13:08:08 -0300
Subject: [PATCH 01/64] CaseInsensitiveDict (deprecate CaselessDict)

---
 scrapy/http/headers.py        | 13 ++++---
 scrapy/pipelines/files.py     |  4 +-
 scrapy/utils/datatypes.py     | 46 +++++++++++++++++++++++
 tests/test_utils_datatypes.py | 71 +++++++++++++++++++++++++----------
 4 files changed, 107 insertions(+), 27 deletions(-)

diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..dfbcf8361 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,5 +1,6 @@
 from w3lib.http import headers_dict_to_raw
-from scrapy.utils.datatypes import CaselessDict
+
+from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
 from scrapy.utils.python import to_unicode
 
 
@@ -76,13 +77,13 @@ class Headers(CaselessDict):
         return headers_dict_to_raw(self)
 
     def to_unicode_dict(self):
-        """ Return headers as a CaselessDict with unicode keys
+        """ Return headers as a CaseInsensitiveDict with unicode keys
         and unicode values. Multiple values are joined with ','.
         """
-        return CaselessDict(
-            (to_unicode(key, encoding=self.encoding),
-             to_unicode(b','.join(value), encoding=self.encoding))
-            for key, value in self.items())
+        return CaseInsensitiveDict(
+            (to_unicode(key, encoding=self.encoding), to_unicode(b','.join(value), encoding=self.encoding))
+            for key, value in self.items()
+        )
 
     def __copy__(self):
         return self.__class__(self)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 13ecd4e6c..2f1a25dfc 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -23,7 +23,7 @@ from scrapy.http import Request
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import md5sum
@@ -143,7 +143,7 @@ class S3FilesStore:
         """ Convert headers to botocore keyword agruments.
         """
         # This is required while we need to support both boto and botocore.
-        mapping = CaselessDict({
+        mapping = CaseInsensitiveDict({
             'Content-Type': 'ContentType',
             'Cache-Control': 'CacheControl',
             'Content-Disposition': 'ContentDisposition',
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index e31284a7f..ca6089e0f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -6,14 +6,30 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import collections
+import warnings
 import weakref
 from collections.abc import Mapping
+from typing import Any, AnyStr
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class CaselessDict(dict):
 
     __slots__ = ()
 
+    def __new__(cls, *args, **kwargs):
+        from scrapy.http.headers import Headers
+
+        if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
+            warnings.warn(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        return super().__new__(cls, *args, **kwargs)
+
     def __init__(self, seq=None):
         super().__init__()
         if seq:
@@ -63,6 +79,36 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
+class CaseInsensitiveDict(collections.UserDict):
+    """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
+
+    It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
+    """
+
+    def __getitem__(self, key: AnyStr) -> Any:
+        return super().__getitem__(self.normkey(key))
+
+    def __setitem__(self, key: AnyStr, value: Any) -> None:
+        super().__setitem__(self.normkey(key), self.normvalue(value))
+
+    def __delitem__(self, key: AnyStr) -> None:
+        super().__delitem__(self.normkey(key))
+
+    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+        return super().__contains__(self.normkey(key))
+
+    def normkey(self, key: AnyStr) -> AnyStr:
+        """Method to normalize dictionary key access"""
+        return key.lower()
+
+    def normvalue(self, value: Any) -> Any:
+        """Method to normalize values prior to be set"""
+        return value
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: {super().__repr__()}>"
+
+
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index e4bccf30e..c033cd537 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,26 +1,34 @@
 import copy
 import unittest
+import warnings
 from collections.abc import Mapping, MutableMapping
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
-from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude
+from scrapy.utils.datatypes import (
+    CaseInsensitiveDict,
+    CaselessDict,
+    LocalCache,
+    LocalWeakReferencedCache,
+    SequenceExclude,
+)
 from scrapy.utils.python import garbage_collect
 
 
 __doctests__ = ['scrapy.utils.datatypes']
 
 
-class CaselessDictTest(unittest.TestCase):
+class CaseInsensitiveDictMixin:
 
     def test_init_dict(self):
         seq = {'red': 1, 'black': 3}
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
     def test_init_pair_sequence(self):
         seq = (('red', 1), ('black', 3))
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
@@ -39,7 +47,7 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
@@ -64,12 +72,12 @@ class CaselessDictTest(unittest.TestCase):
                 return len(self._d)
 
         seq = MyMutableMapping(red=1, black=3)
-        d = CaselessDict(seq)
+        d = self.dict_class(seq)
         self.assertEqual(d['red'], 1)
         self.assertEqual(d['black'], 3)
 
     def test_caseless(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d['key_Lower'] = 1
         self.assertEqual(d['KEy_loWer'], 1)
         self.assertEqual(d.get('KEy_loWer'), 1)
@@ -79,19 +87,19 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get('key_Lower'), 3)
 
     def test_delete(self):
-        d = CaselessDict({'key_lower': 1})
+        d = self.dict_class({'key_lower': 1})
         del d['key_LOWER']
         self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
         self.assertRaises(KeyError, d.__getitem__, 'key_lower')
 
     def test_getdefault(self):
-        d = CaselessDict()
+        d = self.dict_class()
         self.assertEqual(d.get('c', 5), 5)
         d['c'] = 10
         self.assertEqual(d.get('c', 5), 10)
 
     def test_setdefault(self):
-        d = CaselessDict({'a': 1, 'b': 2})
+        d = self.dict_class({'a': 1, 'b': 2})
 
         r = d.setdefault('A', 5)
         self.assertEqual(r, 1)
@@ -104,15 +112,15 @@ class CaselessDictTest(unittest.TestCase):
     def test_fromkeys(self):
         keys = ('a', 'b')
 
-        d = CaselessDict.fromkeys(keys)
+        d = self.dict_class.fromkeys(keys)
         self.assertEqual(d['A'], None)
         self.assertEqual(d['B'], None)
 
-        d = CaselessDict.fromkeys(keys, 1)
+        d = self.dict_class.fromkeys(keys, 1)
         self.assertEqual(d['A'], 1)
         self.assertEqual(d['B'], 1)
 
-        instance = CaselessDict()
+        instance = self.dict_class()
         d = instance.fromkeys(keys)
         self.assertEqual(d['A'], None)
         self.assertEqual(d['B'], None)
@@ -122,18 +130,19 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d['B'], 1)
 
     def test_contains(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d['a'] = 1
         assert 'a' in d
+        assert 'A' in d
 
     def test_pop(self):
-        d = CaselessDict()
+        d = self.dict_class()
         d['a'] = 1
         self.assertEqual(d.pop('A'), 1)
         self.assertRaises(KeyError, d.pop, 'A')
 
     def test_normkey(self):
-        class MyDict(CaselessDict):
+        class MyDict(self.dict_class):
             def normkey(self, key):
                 return key.title()
 
@@ -142,7 +151,7 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(list(d.keys()), ['Key-One'])
 
     def test_normvalue(self):
-        class MyDict(CaselessDict):
+        class MyDict(self.dict_class):
             def normvalue(self, value):
                 if value is not None:
                     return value + 1
@@ -171,11 +180,35 @@ class CaselessDictTest(unittest.TestCase):
         self.assertEqual(d.get('key'), 2)
 
     def test_copy(self):
-        h1 = CaselessDict({'header1': 'value'})
+        h1 = self.dict_class({'header1': 'value'})
         h2 = copy.copy(h1)
         self.assertEqual(h1, h2)
         self.assertEqual(h1.get('header1'), h2.get('header1'))
-        assert isinstance(h2, CaselessDict)
+        assert isinstance(h2, self.dict_class)
+
+
+class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaseInsensitiveDict
+
+    def test_repr(self):
+        d = self.dict_class({"foo": "bar"})
+        self.assertEqual(repr(d), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+
+
+class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+    dict_class = CaselessDict
+
+    def test_deprecation_message(self):
+        with warnings.catch_warnings(record=True) as caught:
+            self.dict_class({"foo": "bar"})
+
+            self.assertEqual(len(caught), 1)
+            self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+            self.assertEqual(
+                "scrapy.utils.datatypes.CaselessDict is deprecated,"
+                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+                str(caught[0].message),
+            )
 
 
 class SequenceExcludeTest(unittest.TestCase):

From bbeed6ae8fd9aed3651b104e4cc3e56495e1b1b9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Aug 2021 14:09:30 -0300
Subject: [PATCH 02/64] CaseInsensitiveDict: preserve original keys (only
 lookups are key-insensitive)

---
 scrapy/utils/datatypes.py     | 36 ++++++++++++++++++++++-------------
 tests/test_utils_datatypes.py | 13 +++++++++++--
 2 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index ca6089e0f..1d56811f0 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -79,35 +79,45 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class CaseInsensitiveDict(collections.UserDict):
+class CaseInsensitiveDict(collections.UserDict,):
     """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
 
     It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
     """
 
+    def __init__(self, *args, **kwargs) -> None:
+        self._keys: dict = {}
+        super().__init__(*args, **kwargs)
+
     def __getitem__(self, key: AnyStr) -> Any:
-        return super().__getitem__(self.normkey(key))
+        normalized_key = self.normkey(key)
+        return super().__getitem__(self._keys[normalized_key.lower()])
 
     def __setitem__(self, key: AnyStr, value: Any) -> None:
-        super().__setitem__(self.normkey(key), self.normvalue(value))
+        normalized_key = self.normkey(key)
+        if normalized_key.lower() in self._keys:
+            del self[self._keys[normalized_key.lower()]]
+        super().__setitem__(normalized_key, self.normvalue(value))
+        self._keys[normalized_key.lower()] = normalized_key
 
     def __delitem__(self, key: AnyStr) -> None:
-        super().__delitem__(self.normkey(key))
+        normalized_key = self.normkey(key)
+        stored_key = self._keys.pop(normalized_key.lower())
+        super().__delitem__(stored_key)
 
     def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
-        return super().__contains__(self.normkey(key))
-
-    def normkey(self, key: AnyStr) -> AnyStr:
-        """Method to normalize dictionary key access"""
-        return key.lower()
-
-    def normvalue(self, value: Any) -> Any:
-        """Method to normalize values prior to be set"""
-        return value
+        normalized_key = self.normkey(key)
+        return normalized_key.lower() in self._keys
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
+    def normkey(self, key: AnyStr) -> AnyStr:
+        return key
+
+    def normvalue(self, value: Any) -> Any:
+        return value
+
 
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index c033cd537..5faaabe81 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,4 +1,5 @@
 import copy
+from typing import Iterator
 import unittest
 import warnings
 from collections.abc import Mapping, MutableMapping
@@ -191,8 +192,16 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
     dict_class = CaseInsensitiveDict
 
     def test_repr(self):
-        d = self.dict_class({"foo": "bar"})
-        self.assertEqual(repr(d), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+        d1 = self.dict_class({"foo": "bar"})
+        self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+        d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        self.assertEqual(repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>")
+
+    def test_iter(self):
+        d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+        iterkeys = iter(d)
+        self.assertIsInstance(iterkeys, Iterator)
+        self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
 
 
 class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):

From 10ebf6384ed58253c237224d523e602b1f3c2224 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Aug 2021 14:12:55 -0300
Subject: [PATCH 03/64] Remove unnecessary comma

---
 scrapy/utils/datatypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 1d56811f0..6eeabe1ee 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -79,7 +79,7 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class CaseInsensitiveDict(collections.UserDict,):
+class CaseInsensitiveDict(collections.UserDict):
     """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
 
     It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.

From 1c031b8a8dd719e6011ee29889bc8181cdbc9a9b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 12 May 2022 13:10:08 -0300
Subject: [PATCH 04/64] Underscore CaseInsensitiveDict normkey/normvalue

---
 scrapy/utils/datatypes.py     | 19 +++++++++----------
 tests/test_utils_datatypes.py | 10 +++++++---
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index f45e1c9b8..807a95504 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -80,9 +80,8 @@ class CaselessDict(dict):
 
 
 class CaseInsensitiveDict(collections.UserDict):
-    """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups.
-
-    It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods.
+    """A dict-like structure that accepts strings or bytes
+    as keys and allows case-insensitive lookups.
     """
 
     def __init__(self, *args, **kwargs) -> None:
@@ -90,32 +89,32 @@ class CaseInsensitiveDict(collections.UserDict):
         super().__init__(*args, **kwargs)
 
     def __getitem__(self, key: AnyStr) -> Any:
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         return super().__getitem__(self._keys[normalized_key.lower()])
 
     def __setitem__(self, key: AnyStr, value: Any) -> None:
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         if normalized_key.lower() in self._keys:
             del self[self._keys[normalized_key.lower()]]
-        super().__setitem__(normalized_key, self.normvalue(value))
+        super().__setitem__(normalized_key, self._normvalue(value))
         self._keys[normalized_key.lower()] = normalized_key
 
     def __delitem__(self, key: AnyStr) -> None:
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         stored_key = self._keys.pop(normalized_key.lower())
         super().__delitem__(stored_key)
 
     def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
-        normalized_key = self.normkey(key)
+        normalized_key = self._normkey(key)
         return normalized_key.lower() in self._keys
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
-    def normkey(self, key: AnyStr) -> AnyStr:
+    def _normkey(self, key: AnyStr) -> AnyStr:
         return key
 
-    def normvalue(self, value: Any) -> Any:
+    def _normvalue(self, value: Any) -> Any:
         return value
 
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 5faaabe81..0a724f237 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,8 +1,8 @@
 import copy
-from typing import Iterator
 import unittest
 import warnings
 from collections.abc import Mapping, MutableMapping
+from typing import Iterator
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
@@ -144,19 +144,23 @@ class CaseInsensitiveDictMixin:
 
     def test_normkey(self):
         class MyDict(self.dict_class):
-            def normkey(self, key):
+            def _normkey(self, key):
                 return key.title()
 
+            normkey = _normkey  # deprecated CaselessDict class
+
         d = MyDict()
         d['key-one'] = 2
         self.assertEqual(list(d.keys()), ['Key-One'])
 
     def test_normvalue(self):
         class MyDict(self.dict_class):
-            def normvalue(self, value):
+            def _normvalue(self, value):
                 if value is not None:
                     return value + 1
 
+            normvalue = _normvalue  # deprecated CaselessDict class
+
         d = MyDict({'key': 1})
         self.assertEqual(d['key'], 2)
         self.assertEqual(d.get('key'), 2)

From 2c65066ad9e293630da2c594af06ad483abe800d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 27 May 2022 19:56:42 -0300
Subject: [PATCH 05/64] Avoid exceptions on copy

---
 scrapy/utils/datatypes.py     | 7 +++++--
 tests/test_utils_datatypes.py | 8 +++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 807a95504..fd5ac3b08 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -94,8 +94,11 @@ class CaseInsensitiveDict(collections.UserDict):
 
     def __setitem__(self, key: AnyStr, value: Any) -> None:
         normalized_key = self._normkey(key)
-        if normalized_key.lower() in self._keys:
-            del self[self._keys[normalized_key.lower()]]
+        try:
+            lower_key = self._keys[normalized_key.lower()]
+            del self[lower_key]
+        except KeyError:
+            pass
         super().__setitem__(normalized_key, self._normvalue(value))
         self._keys[normalized_key.lower()] = normalized_key
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 0a724f237..36df9006f 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -187,9 +187,15 @@ class CaseInsensitiveDictMixin:
     def test_copy(self):
         h1 = self.dict_class({'header1': 'value'})
         h2 = copy.copy(h1)
+        assert isinstance(h2, self.dict_class)
         self.assertEqual(h1, h2)
         self.assertEqual(h1.get('header1'), h2.get('header1'))
-        assert isinstance(h2, self.dict_class)
+        self.assertEqual(h1.get('header1'), h2.get('HEADER1'))
+        h3 = h1.copy()
+        assert isinstance(h3, self.dict_class)
+        self.assertEqual(h1, h3)
+        self.assertEqual(h1.get('header1'), h3.get('header1'))
+        self.assertEqual(h1.get('header1'), h3.get('HEADER1'))
 
 
 class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):

From a34b929a40c12933f75db4665b71348444a5d603 Mon Sep 17 00:00:00 2001
From: srki24 <srdjan,djuric@outlook.com>
Date: Fri, 4 Nov 2022 18:00:17 +0100
Subject: [PATCH 06/64] issues/5043 Detaching the stream

---
 scrapy/exporters.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 76cbe4d4b..243ec4fe1 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -247,6 +247,12 @@ class CsvItemExporter(BaseItemExporter):
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
+    def finish_exporting(self):
+        # Detaching stream in order to avoid file closing.
+        # The file will be closed with slot.storage.store
+        # https://github.com/scrapy/scrapy/issues/5043
+        self.stream.detach()
+
     def _build_row(self, values):
         for s in values:
             try:

From 2f2bcb006d349eeeed10018362c780496be96550 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 2 Feb 2023 05:55:59 +0100
Subject: [PATCH 07/64] Test stream detaching in CsvItemExporter

---
 scrapy/exporters.py     |  5 +----
 tests/test_exporters.py | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 243ec4fe1..42105690c 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -248,10 +248,7 @@ class CsvItemExporter(BaseItemExporter):
         self.csv_writer.writerow(values)
 
     def finish_exporting(self):
-        # Detaching stream in order to avoid file closing.
-        # The file will be closed with slot.storage.store
-        # https://github.com/scrapy/scrapy/issues/5043
-        self.stream.detach()
+        self.stream.detach()  # Avoid closing the wrapped file.
 
     def _build_row(self, values):
         for s in values:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 69ac928c3..bec8d2267 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -85,6 +85,10 @@ class BaseItemExporterTest(unittest.TestCase):
             if self.ie.__class__ is not BaseItemExporter:
                 raise
         self.ie.finish_exporting()
+        # Delete the item exporter object, so that if it causes the output
+        # file handle be closed, which should not be the case, follow-up
+        # interactions with the output file handle will surface the issue.
+        del self.ie
         self._check_output()
 
     def test_export_item(self):
@@ -230,6 +234,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.export_item(i1)
         ie.export_item(i2)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         f.seek(0)
         self.assertEqual(self.item_class(**pickle.load(f)), i1)
         self.assertEqual(self.item_class(**pickle.load(f)), i2)
@@ -241,6 +246,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         self.assertEqual(pickle.loads(fp.getvalue()), item)
 
 
@@ -267,6 +273,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         fp.seek(0)
         self.assertEqual(marshal.load(fp), item)
 
@@ -299,6 +306,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         self.assertCsvEqual(fp.getvalue(), expected)
 
     def test_header_export_all(self):
@@ -330,6 +338,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             ie.export_item(item)
             ie.export_item(item)
             ie.finish_exporting()
+            del ie  # See the first “del self.ie” in this file for context.
             self.assertCsvEqual(output.getvalue(),
                                 b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
@@ -414,6 +423,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         ie.start_exporting()
         ie.export_item(item)
         ie.finish_exporting()
+        del ie  # See the first “del self.ie” in this file for context.
         self.assertXmlEquivalent(fp.getvalue(), expected_value)
 
     def _check_output(self):
@@ -520,6 +530,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         self.assertEqual(exported, self._expected_nested)
 
@@ -534,6 +545,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(item)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         item['time'] = str(item['time'])
         self.assertEqual(exported, item)
@@ -561,6 +573,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.export_item(item)
         self.ie.export_item(item)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
 
@@ -577,6 +590,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
         self.assertEqual(exported, [expected])
@@ -588,6 +602,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(i3)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
         self.assertEqual(exported, [expected])
@@ -597,6 +612,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
         self.ie.start_exporting()
         self.ie.export_item(item)
         self.ie.finish_exporting()
+        del self.ie  # See the first “del self.ie” in this file for context.
         exported = json.loads(to_unicode(self.output.getvalue()))
         item['time'] = str(item['time'])
         self.assertEqual(exported, [item])

From 426f3ebb7b368084f6e77ccf8a121c85c7913049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 2 Feb 2023 05:58:32 +0100
Subject: [PATCH 08/64] =?UTF-8?q?Fix=20typo:=20causes=20it=20be=20closed?=
 =?UTF-8?q?=20=E2=86=92=20causes=20it=20to=20be=20closed?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_exporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 34475b05d..95ff5a93c 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -92,7 +92,7 @@ class BaseItemExporterTest(unittest.TestCase):
                 raise
         self.ie.finish_exporting()
         # Delete the item exporter object, so that if it causes the output
-        # file handle be closed, which should not be the case, follow-up
+        # file handle to be closed, which should not be the case, follow-up
         # interactions with the output file handle will surface the issue.
         del self.ie
         self._check_output()

From 60bf56b715e443951125a10ff91ad1699270a82c Mon Sep 17 00:00:00 2001
From: Yegor <yahor.statkevich@godeltech.com>
Date: Wed, 15 Feb 2023 12:15:24 +0100
Subject: [PATCH 09/64] Add boto3 availability util method

---
 scrapy/utils/boto.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 085ee7d25..7b18b6bcf 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -8,3 +8,12 @@ def is_botocore_available():
         return True
     except ImportError:
         return False
+
+
+def is_boto3_available():
+    try:
+        import boto3  # noqa: F401
+
+        return True
+    except ImportError:
+        return False

From 01ad49515d31e3053949d20e923dc61a87147eab Mon Sep 17 00:00:00 2001
From: Yegor <yahor.statkevich@godeltech.com>
Date: Wed, 15 Feb 2023 12:18:28 +0100
Subject: [PATCH 10/64] Use boto3 session and client

---
 scrapy/extensions/feedexport.py | 37 ++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index cd26b5778..34eb8e4b4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -173,16 +173,37 @@ class S3FeedStorage(BlockingFeedStorage):
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
         self.endpoint_url = endpoint_url
-        import botocore.session
+        if is_boto3_available():
+            import boto3.session
+            session = boto3.session.Session()
+
+            self.s3_client = session.client(
+                "s3",
+                aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key,
+                aws_session_token=self.session_token,
+                endpoint_url=self.endpoint_url,
+            )
+        else:
+            warnings.warn(
+                "Botocore usage is deprecated for S3FeedStorage, "
+                "please use boto3 to avoid problems",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+
+            import botocore.session
+            session = botocore.get_session()
 
         session = botocore.session.get_session()
-        self.s3_client = session.create_client(
-            "s3",
-            aws_access_key_id=self.access_key,
-            aws_secret_access_key=self.secret_key,
-            aws_session_token=self.session_token,
-            endpoint_url=self.endpoint_url,
-        )
+            self.s3_client = session.create_client(
+                "s3",
+                aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key,
+                aws_session_token=self.session_token,
+                endpoint_url=self.endpoint_url,
+            )
+
         if feed_options and feed_options.get("overwrite", True) is False:
             logger.warning(
                 "S3 does not support appending to files. To "

From c1a8baa1fa0c6210e5c1351ab5a4063b8dacaa7b Mon Sep 17 00:00:00 2001
From: Yegor <yahor.statkevich@godeltech.com>
Date: Wed, 15 Feb 2023 12:20:01 +0100
Subject: [PATCH 11/64] Add forgotten import

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 34eb8e4b4..0e05337bc 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -22,6 +22,7 @@ from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
+from scrapy.utils.boto import is_boto3_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
@@ -195,7 +196,6 @@ class S3FeedStorage(BlockingFeedStorage):
             import botocore.session
             session = botocore.get_session()
 
-        session = botocore.session.get_session()
             self.s3_client = session.create_client(
                 "s3",
                 aws_access_key_id=self.access_key,

From 59ba3c4e4cabd661810e0ac1b9963040d2e42f31 Mon Sep 17 00:00:00 2001
From: Yegor <yahor.statkevich@godeltech.com>
Date: Wed, 15 Feb 2023 16:29:06 +0100
Subject: [PATCH 12/64] Use boto3's `upload_fileobj`

---
 scrapy/extensions/feedexport.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0e05337bc..601fde7eb 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -174,7 +174,9 @@ class S3FeedStorage(BlockingFeedStorage):
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
         self.endpoint_url = endpoint_url
-        if is_boto3_available():
+        self._using_boto3 = is_boto3_available()
+
+        if self._using_boto3:
             import boto3.session
             session = boto3.session.Session()
 
@@ -187,8 +189,8 @@ class S3FeedStorage(BlockingFeedStorage):
             )
         else:
             warnings.warn(
-                "Botocore usage is deprecated for S3FeedStorage, "
-                "please use boto3 to avoid problems",
+                "`botocore` usage has been deprecated for S3 feed "
+                "export, please use `boto3` to avoid problems",
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )
@@ -227,9 +229,14 @@ class S3FeedStorage(BlockingFeedStorage):
     def _store_in_thread(self, file):
         file.seek(0)
         kwargs = {"ACL": self.acl} if self.acl else {}
-        self.s3_client.put_object(
-            Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
-        )
+        if self._using_boto3:
+            self.s3_client.upload_fileobj(
+                Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
+            )
+        else:
+            self.s3_client.put_object(
+                Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
+            )
         file.close()
 
 

From 29c2477f0a8365d0476fb8c07f391fa109f4615a Mon Sep 17 00:00:00 2001
From: Yegor <yahor.statkevich@godeltech.com>
Date: Wed, 15 Feb 2023 16:40:05 +0100
Subject: [PATCH 13/64] Document the need to install boto3

---
 docs/topics/feed-exports.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index eef0bb5ca..8aa3e3be4 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -101,12 +101,12 @@ The storages backends supported out of the box are:
 
 -   :ref:`topics-feed-storage-fs`
 -   :ref:`topics-feed-storage-ftp`
--   :ref:`topics-feed-storage-s3` (requires botocore_)
+-   :ref:`topics-feed-storage-s3` (requires boto3_)
 -   :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
 -   :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if the botocore_
+not available. For example, the S3 backend is only available if at least the botocore_
 library is installed.
 
 
@@ -193,7 +193,7 @@ The feeds are stored on `Amazon S3`_.
 
     -   ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
--   Required external libraries: `botocore`_ >= 1.4.87
+-   Required external libraries: `boto3`_ >= 1.26.70, will fall back to botocore_ if unavailable
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -779,6 +779,7 @@ source spider in the feed URI:
 
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
+.. _boto3: https://github.com/boto/boto3
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Google Cloud Storage: https://cloud.google.com/storage/

From f45a7d3f3c90ad27b2b13bd1ee038ce2fe9c02f7 Mon Sep 17 00:00:00 2001
From: Yegor <yahor.statkevich@godeltech.com>
Date: Wed, 15 Feb 2023 17:07:55 +0100
Subject: [PATCH 14/64] Remove `stacklevel` from warning

---
 scrapy/extensions/feedexport.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 601fde7eb..e27e6f915 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -192,7 +192,6 @@ class S3FeedStorage(BlockingFeedStorage):
                 "`botocore` usage has been deprecated for S3 feed "
                 "export, please use `boto3` to avoid problems",
                 category=ScrapyDeprecationWarning,
-                stacklevel=2,
             )
 
             import botocore.session

From eb0cca471d20f00a1cc0980b84fe6266772faaa3 Mon Sep 17 00:00:00 2001
From: jazzthief <mynameisyegor@gmail.com>
Date: Thu, 16 Feb 2023 16:53:49 +0100
Subject: [PATCH 15/64] Apply pre-commit changes

---
 scrapy/extensions/feedexport.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e27e6f915..36107a62b 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -21,8 +21,7 @@ from zope.interface import Interface, implementer
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.boto import is_boto3_available
+from scrapy.utils.boto import is_boto3_available, is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
@@ -178,6 +177,7 @@ class S3FeedStorage(BlockingFeedStorage):
 
         if self._using_boto3:
             import boto3.session
+
             session = boto3.session.Session()
 
             self.s3_client = session.client(
@@ -195,6 +195,7 @@ class S3FeedStorage(BlockingFeedStorage):
             )
 
             import botocore.session
+
             session = botocore.get_session()
 
             self.s3_client = session.create_client(

From 45b9dbae40de00fc7d4498e871538c074c64be8f Mon Sep 17 00:00:00 2001
From: jazzthief <mynameisyegor@gmail.com>
Date: Wed, 22 Feb 2023 13:28:34 +0100
Subject: [PATCH 16/64] Fix a typo

---
 scrapy/extensions/feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 36107a62b..c4ec410e3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -196,7 +196,7 @@ class S3FeedStorage(BlockingFeedStorage):
 
             import botocore.session
 
-            session = botocore.get_session()
+            session = botocore.session.get_session()
 
             self.s3_client = session.create_client(
                 "s3",

From 39dbfa1d8276e1c8abb6aedc51f644ee27264f90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 15 Mar 2023 15:31:39 +0100
Subject: [PATCH 17/64] Minimize test reliance on S3; do not install botocore
 on the default test environments

---
 tests/test_feedexport.py     | 23 ++++++-----------------
 tests/test_pipeline_files.py | 12 ++++++++----
 tox.ini                      |  4 +---
 3 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eafe1b334..19ca311c3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -235,8 +235,10 @@ class BlockingFeedStorageTest(unittest.TestCase):
 
 
 class S3FeedStorageTest(unittest.TestCase):
-    def test_parse_credentials(self):
+    def setUp(self):
         skip_if_no_boto()
+
+    def test_parse_credentials(self):
         aws_credentials = {
             "AWS_ACCESS_KEY_ID": "settings_key",
             "AWS_SECRET_ACCESS_KEY": "settings_secret",
@@ -272,8 +274,6 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store(self):
-        skip_if_no_boto()
-
         settings = {
             "AWS_ACCESS_KEY_ID": "access_key",
             "AWS_SECRET_ACCESS_KEY": "secret_key",
@@ -392,7 +392,6 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
-        skip_if_no_boto()
         storage = S3FeedStorage(
             "s3://mybucket/export.csv",
             "access_key",
@@ -408,7 +407,6 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
-        skip_if_no_boto()
         storage = S3FeedStorage(
             "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
         )
@@ -888,15 +886,10 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_stats_multiple_file(self):
         settings = {
-            "AWS_ACCESS_KEY_ID": "access_key",
-            "AWS_SECRET_ACCESS_KEY": "secret_key",
             "FEEDS": {
                 printf_escape(path_to_url(str(self._random_temp_filename()))): {
                     "format": "json",
                 },
-                "s3://bucket/key/foo.csv": {
-                    "format": "csv",
-                },
                 "stdout:": {
                     "format": "xml",
                 },
@@ -908,18 +901,12 @@ class FeedExportTest(FeedExportTestBase):
         self.assertIn(
             "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()
         )
-        self.assertIn(
-            "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()
-        )
         self.assertIn(
             "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()
         )
         self.assertEqual(
             crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1
         )
-        self.assertEqual(
-            crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1
-        )
         self.assertEqual(
             crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1
         )
@@ -2535,7 +2522,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_s3_export(self):
         skip_if_no_boto()
-
         bucket = "mybucket"
         items = [
             self.MyItem({"foo": "bar1", "egg": "spam1"}),
@@ -2707,6 +2693,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     maxDiff = None
 
+    def setUp(self):
+        skip_if_no_boto()
+
     def test_init(self):
         settings_dict = {
             "FEED_URI": "file:///tmp/foobar",
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 9701e5d4e..e0bcfcfea 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -225,12 +225,16 @@ class FilesPipelineTestCase(unittest.TestCase):
 
 
 class FilesPipelineTestCaseFieldsMixin:
+    def setUp(self):
+        self.tempdir = mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.tempdir)
+
     def test_item_fields_default(self):
         url = "http://www.example.com/files/1.txt"
         item = self.item_class(name="item1", file_urls=[url])
-        pipeline = FilesPipeline.from_settings(
-            Settings({"FILES_STORE": "s3://example/files/"})
-        )
+        pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
         requests = list(pipeline.get_media_requests(item, None))
         self.assertEqual(requests[0].url, url)
         results = [(True, {"url": url})]
@@ -245,7 +249,7 @@ class FilesPipelineTestCaseFieldsMixin:
         pipeline = FilesPipeline.from_settings(
             Settings(
                 {
-                    "FILES_STORE": "s3://example/files/",
+                    "FILES_STORE": self.tempdir,
                     "FILES_URLS_FIELD": "custom_file_urls",
                     "FILES_RESULT_FIELD": "custom_files",
                 }
diff --git a/tox.ini b/tox.ini
index 5a9d9cf29..5c2f583d9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,8 +18,6 @@ deps =
     mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
     # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
     markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
-    # Extras
-    botocore>=1.4.87
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -121,7 +119,7 @@ setenv =
 basepython = python3
 deps =
     {[testenv]deps}
-    boto
+    botocore>=1.4.87
     google-cloud-storage
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.

From 590955fac8de5d1f951b4ecb724eb2ea4f212653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 15 Mar 2023 16:03:44 +0100
Subject: [PATCH 18/64] Provide separate test environments for botocore and
 boto3 extras; add extra-deps-pinned and remote extras from pinned

---
 .github/workflows/tests-ubuntu.yml |  9 ++++++
 tox.ini                            | 44 ++++++++++++++++++++++++------
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 8fcf90a18..96b26a1f8 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -37,10 +37,19 @@ jobs:
         - python-version: pypy3.7
           env:
             TOXENV: pypy3-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: extra-deps-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: botocore-pinned
 
         - python-version: "3.11"
           env:
             TOXENV: extra-deps
+        - python-version: "3.11"
+          env:
+            TOXENV: botocore
 
     steps:
     - uses: actions/checkout@v3
diff --git a/tox.ini b/tox.ini
index 5c2f583d9..f94d7f751 100644
--- a/tox.ini
+++ b/tox.ini
@@ -88,11 +88,6 @@ deps =
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
-
-    # Extras
-    botocore==1.4.87
-    google-cloud-storage==1.29.0
-    Pillow==7.1.0
 setenv =
     _SCRAPY_PINNED=true
 install_command =
@@ -119,14 +114,26 @@ setenv =
 basepython = python3
 deps =
     {[testenv]deps}
-    botocore>=1.4.87
+    boto3
     google-cloud-storage
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
     robotexclusionrulesparser
-    Pillow>=4.0.0
-    Twisted[http2]>=17.9.0
+    Pillow
+    Twisted[http2]
+
+[testenv:extra-deps-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    boto3==1.0.0
+    google-cloud-storage==1.29.0
+    Pillow==7.1.0
+    robotexclusionrulesparser==1.6.2
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
 
 [testenv:asyncio]
 commands =
@@ -185,3 +192,24 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+
+# Run S3 tests with botocore installed but without boto3.
+
+[testenv:botocore]
+deps =
+    {[testenv]deps}
+    botocore>=1.4.87
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[testenv:botocore-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    botocore==1.4.87
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}

From 7e7b41c6b32a639395cf3183e98fa8a8a78afd98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 15 Mar 2023 16:38:13 +0100
Subject: [PATCH 19/64] Fix test expectations for boto3

---
 scrapy/extensions/feedexport.py |  3 +-
 tests/test_feedexport.py        | 72 +++++++++++++++++++++------------
 2 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c4ec410e3..4f0a946de 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -228,12 +228,13 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file):
         file.seek(0)
-        kwargs = {"ACL": self.acl} if self.acl else {}
         if self._using_boto3:
+            kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
             self.s3_client.upload_fileobj(
                 Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
             )
         else:
+            kwargs = {"ACL": self.acl} if self.acl else {}
             self.s3_client.put_object(
                 Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
             )
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 19ca311c3..2e350df65 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -44,6 +44,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage,
     StdoutFeedStorage,
     _FeedSlot,
+    is_boto3_available,
 )
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
@@ -285,30 +286,39 @@ class S3FeedStorageTest(unittest.TestCase):
         verifyObject(IFeedStorage, storage)
 
         file = mock.MagicMock()
-        from botocore.stub import Stubber
-
-        with Stubber(storage.s3_client) as stub:
-            stub.add_response(
-                "put_object",
-                expected_params={
-                    "Body": file,
-                    "Bucket": bucket,
-                    "Key": key,
-                },
-                service_response={},
-            )
 
+        if is_boto3_available():
+            storage.s3_client = mock.MagicMock()
             yield storage.store(file)
-
-            stub.assert_no_pending_responses()
             self.assertEqual(
-                file.method_calls,
-                [
-                    mock.call.seek(0),
-                    # The call to read does not happen with Stubber
-                    mock.call.close(),
-                ],
+                storage.s3_client.upload_fileobj.call_args,
+                mock.call(Bucket=bucket, Key=key, Fileobj=file),
             )
+        else:
+            from botocore.stub import Stubber
+
+            with Stubber(storage.s3_client) as stub:
+                stub.add_response(
+                    "put_object",
+                    expected_params={
+                        "Body": file,
+                        "Bucket": bucket,
+                        "Key": key,
+                    },
+                    service_response={},
+                )
+
+                yield storage.store(file)
+
+                stub.assert_no_pending_responses()
+                self.assertEqual(
+                    file.method_calls,
+                    [
+                        mock.call.seek(0),
+                        # The call to read does not happen with Stubber
+                        mock.call.close(),
+                    ],
+                )
 
     def test_init_without_acl(self):
         storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@@ -391,7 +401,7 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.endpoint_url, "https://example.com")
 
     @defer.inlineCallbacks
-    def test_store_botocore_without_acl(self):
+    def test_store_without_acl(self):
         storage = S3FeedStorage(
             "s3://mybucket/export.csv",
             "access_key",
@@ -403,10 +413,18 @@ class S3FeedStorageTest(unittest.TestCase):
 
         storage.s3_client = mock.MagicMock()
         yield storage.store(BytesIO(b"test file"))
-        self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1])
+        if is_boto3_available():
+            acl = (
+                storage.s3_client.upload_fileobj.call_args[1]
+                .get("ExtraArgs", {})
+                .get("ACL")
+            )
+        else:
+            acl = storage.s3_client.put_object.call_args[1].get("ACL")
+        self.assertIsNone(acl)
 
     @defer.inlineCallbacks
-    def test_store_botocore_with_acl(self):
+    def test_store_with_acl(self):
         storage = S3FeedStorage(
             "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
         )
@@ -416,9 +434,11 @@ class S3FeedStorageTest(unittest.TestCase):
 
         storage.s3_client = mock.MagicMock()
         yield storage.store(BytesIO(b"test file"))
-        self.assertEqual(
-            storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl"
-        )
+        if is_boto3_available():
+            acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
+        else:
+            acl = storage.s3_client.put_object.call_args[1]["ACL"]
+        self.assertEqual(acl, "custom-acl")
 
     def test_overwrite_default(self):
         with LogCapture() as log:

From a17d996da2dad6d250dd93da34b5b63f3d63239d Mon Sep 17 00:00:00 2001
From: jazzthief <mynameisyegor@gmail.com>
Date: Thu, 16 Mar 2023 17:20:22 +0100
Subject: [PATCH 20/64] Change `boto3` version to 1.20.0 for
 `extra-deps-pinned` env

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index f94d7f751..80fc28735 100644
--- a/tox.ini
+++ b/tox.ini
@@ -127,7 +127,7 @@ deps =
 basepython = python3.7
 deps =
     {[pinned]deps}
-    boto3==1.0.0
+    boto3==1.20.0
     google-cloud-storage==1.29.0
     Pillow==7.1.0
     robotexclusionrulesparser==1.6.2

From 4ebc08ef1042cafd16c6a7eb20e3a4dcf43e3c97 Mon Sep 17 00:00:00 2001
From: jazzthief <mynameisyegor@gmail.com>
Date: Thu, 16 Mar 2023 17:24:11 +0100
Subject: [PATCH 21/64] Switch from `is_boto3_available()` to
 `IS_BOTO3_AVAILABLE` var

---
 scrapy/extensions/feedexport.py | 14 ++++++++++----
 scrapy/utils/boto.py            |  9 ---------
 tests/test_feedexport.py        |  8 ++++----
 3 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4f0a946de..83849ca13 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -21,7 +21,7 @@ from zope.interface import Interface, implementer
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.utils.boto import is_boto3_available, is_botocore_available
+from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
@@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values
 
 logger = logging.getLogger(__name__)
 
+try:
+    import boto3  # noqa: F401
+
+    IS_BOTO3_AVAILABLE = True
+except ImportError:
+    IS_BOTO3_AVAILABLE = False
+
 
 def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
     argument_names = get_func_args(builder)
@@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage):
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
         self.endpoint_url = endpoint_url
-        self._using_boto3 = is_boto3_available()
 
-        if self._using_boto3:
+        if IS_BOTO3_AVAILABLE:
             import boto3.session
 
             session = boto3.session.Session()
@@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file):
         file.seek(0)
-        if self._using_boto3:
+        if IS_BOTO3_AVAILABLE:
             kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
             self.s3_client.upload_fileobj(
                 Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 7b18b6bcf..085ee7d25 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -8,12 +8,3 @@ def is_botocore_available():
         return True
     except ImportError:
         return False
-
-
-def is_boto3_available():
-    try:
-        import boto3  # noqa: F401
-
-        return True
-    except ImportError:
-        return False
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2e350df65..7df3e6dd3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -35,6 +35,7 @@ import scrapy
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.exporters import CsvItemExporter, JsonItemExporter
 from scrapy.extensions.feedexport import (
+    IS_BOTO3_AVAILABLE,
     BlockingFeedStorage,
     FeedExporter,
     FileFeedStorage,
@@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage,
     StdoutFeedStorage,
     _FeedSlot,
-    is_boto3_available,
 )
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
@@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase):
 
         file = mock.MagicMock()
 
-        if is_boto3_available():
+        if IS_BOTO3_AVAILABLE:
             storage.s3_client = mock.MagicMock()
             yield storage.store(file)
             self.assertEqual(
@@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase):
 
         storage.s3_client = mock.MagicMock()
         yield storage.store(BytesIO(b"test file"))
-        if is_boto3_available():
+        if IS_BOTO3_AVAILABLE:
             acl = (
                 storage.s3_client.upload_fileobj.call_args[1]
                 .get("ExtraArgs", {})
@@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase):
 
         storage.s3_client = mock.MagicMock()
         yield storage.store(BytesIO(b"test file"))
-        if is_boto3_available():
+        if IS_BOTO3_AVAILABLE:
             acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
         else:
             acl = storage.s3_client.put_object.call_args[1]["ACL"]

From cb67bc17b72a1ae619c89cc26d6f526dd2a26338 Mon Sep 17 00:00:00 2001
From: jazzthief <mynameisyegor@gmail.com>
Date: Thu, 16 Mar 2023 17:25:05 +0100
Subject: [PATCH 22/64] Remove `botocore` from docs

---
 docs/topics/feed-exports.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 8aa3e3be4..5eea6aaf9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -106,7 +106,7 @@ The storages backends supported out of the box are:
 -   :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if at least the botocore_
+not available. For example, the S3 backend is only available if the boto3_
 library is installed.
 
 
@@ -193,7 +193,7 @@ The feeds are stored on `Amazon S3`_.
 
     -   ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
--   Required external libraries: `boto3`_ >= 1.26.70, will fall back to botocore_ if unavailable
+-   Required external libraries: `boto3`_ >= 1.20.0
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -780,6 +780,5 @@ source spider in the feed URI:
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _boto3: https://github.com/boto/boto3
-.. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Google Cloud Storage: https://cloud.google.com/storage/

From 67bfb304cdeb19a0b72b0a09542582f718cf07d6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 21 Apr 2023 18:52:58 +0400
Subject: [PATCH 23/64] Release notes for the current master.

---
 docs/news.rst            | 105 ++++++++++++++++++++++++++++++++++++++-
 docs/topics/settings.rst |   2 +-
 2 files changed, 104 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 9b9eeac71..6cf366449 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,108 @@
 Release notes
 =============
 
+.. _release-2.9.0:
+
+Scrapy 2.9.0 (YYYY-MM-DD)
+-------------------------
+
+Highlights:
+
+-   Per-domain request settings.
+-   Compatibility with new cryptography_ and new parsel_.
+-   TBD
+
+New features
+~~~~~~~~~~~~
+
+-   Settings correponding to :setting:`DOWNLOAD_DELAY`,
+    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
+    :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis
+    via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
+
+-   Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a
+    curl command from a :class:`~scrapy.Request` object. (:issue:`5892`)
+
+-   Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be
+    :class:`pathlib.Path` instances. (:issue:`5801`)
+
+-   :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed
+    string values for the curl ``--data-raw`` argument, which are produced by
+    browsers for data that includes certain symbols. (:issue:`5899`,
+    :issue:`5901`)
+
+-   The ``scrapy parse`` command now also works with async generator callbacks.
+    (:issue:`5819`, :issue:`5824`)
+
+-   The ``scrapy genspider`` command now properly works with HTTPS URLs.
+    (:issue:`3553`, :issue:`5808`)
+
+-   Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`)
+
+-   :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    now skips certain malformed URLs instead of raising an exception.
+    (:issue:`5881`)
+
+-   :func:`scrapy.utils.python.get_func_args` now supports more types of
+    callables. (:issue:`5872`, :issue:`5885`)
+
+Bug fixes
+~~~~~~~~~
+
+-   Fixed an error when using feed postprocessing with S3 storage.
+    (:issue:`5500`, :issue:`5581`)
+
+-   Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method.
+    (:issue:`5811`, :issue:`5821`)
+
+-   Fixed an error when using cryptography_ 40.0.0+ and
+    :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled.
+    (:issue:`5857`, :issue:`5858`)
+
+-   The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline`
+    for files on Google Cloud Storage are no longer Base64-encoded.
+    (:issue:`5874`, :issue:`5891`)
+
+-   Fixed an error breaking user handling of send failures in
+    :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Expanded contributing docs. (:issue:`5109`, :issue:`5851`)
+
+-   Added blacken-docs_ to pre-commit and reformatted the docs with it.
+    (:issue:`5813`, :issue:`5816`)
+
+-   Fixed a JS issue. (:issue:`5875`, :issue:`5877`)
+
+-   Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`)
+
+-   Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`,
+    :issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`)
+
+-   Tests for most of the examples in the docs are now run as a part of CI,
+    found problems were fixed. (:issue:`5816`, :issue:`5826`)
+
+-   Removed usage of deprecated Python classes. (:issue:`5849`)
+
+-   Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`)
+
+-   Fixed a random failure of the ``test_feedexport.test_batch_path_differ``
+    test. (:issue:`5855`, :issue:`5898`)
+
+-   Updated docstrings to match output produced by parsel_ 1.8.1 so that they
+    don't cause test failures. (:issue:`5902`)
+
+-   Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`)
+
+.. _blacken-docs: https://github.com/adamchainz/blacken-docs
+
 .. _release-2.8.0:
 
 Scrapy 2.8.0 (2023-02-02)
@@ -4207,8 +4309,6 @@ Relocations
   + Note: telnet is not enabled on Python 3
     (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595)
 
-.. _parsel: https://github.com/scrapy/parsel
-
 
 Bugfixes
 ~~~~~~~~
@@ -5638,6 +5738,7 @@ First release of Scrapy.
 .. _LevelDB: https://github.com/google/leveldb
 .. _lxml: https://lxml.de/
 .. _marshal: https://docs.python.org/2/library/marshal.html
+.. _parsel: https://github.com/scrapy/parsel
 .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
 .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
 .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4412b5c1c..3e06d84f9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -783,7 +783,7 @@ DOWNLOAD_SLOTS
 
 Default: ``{}``
 
-Allows to define concurrency/delay parameters on per slot(domain) basis:
+Allows to define concurrency/delay parameters on per slot (domain) basis:
 
     .. code-block:: python
 

From d1d6465ef4ea8987efb08e2f9abfd65b36719e04 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 4 May 2023 17:19:01 +0400
Subject: [PATCH 24/64] Address feedback.

---
 docs/news.rst | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 6cf366449..5f189760d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -10,7 +10,7 @@ Scrapy 2.9.0 (YYYY-MM-DD)
 
 Highlights:
 
--   Per-domain request settings.
+-   Per-domain download settings.
 -   Compatibility with new cryptography_ and new parsel_.
 -   TBD
 
@@ -19,7 +19,7 @@ New features
 
 -   Settings correponding to :setting:`DOWNLOAD_DELAY`,
     :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
-    :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis
+    :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
     via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
 
 -   Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a
@@ -28,26 +28,6 @@ New features
 -   Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be
     :class:`pathlib.Path` instances. (:issue:`5801`)
 
--   :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed
-    string values for the curl ``--data-raw`` argument, which are produced by
-    browsers for data that includes certain symbols. (:issue:`5899`,
-    :issue:`5901`)
-
--   The ``scrapy parse`` command now also works with async generator callbacks.
-    (:issue:`5819`, :issue:`5824`)
-
--   The ``scrapy genspider`` command now properly works with HTTPS URLs.
-    (:issue:`3553`, :issue:`5808`)
-
--   Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`)
-
--   :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
-    now skips certain malformed URLs instead of raising an exception.
-    (:issue:`5881`)
-
--   :func:`scrapy.utils.python.get_func_args` now supports more types of
-    callables. (:issue:`5872`, :issue:`5885`)
-
 Bug fixes
 ~~~~~~~~~
 
@@ -65,6 +45,26 @@ Bug fixes
     for files on Google Cloud Storage are no longer Base64-encoded.
     (:issue:`5874`, :issue:`5891`)
 
+-   :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed
+    string values for the curl ``--data-raw`` argument, which are produced by
+    browsers for data that includes certain symbols. (:issue:`5899`,
+    :issue:`5901`)
+
+-   The :command:`parse` command now also works with async generator callbacks.
+    (:issue:`5819`, :issue:`5824`)
+
+-   The :command:`genspider` command now properly works with HTTPS URLs.
+    (:issue:`3553`, :issue:`5808`)
+
+-   Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`)
+
+-   :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    now skips certain malformed URLs instead of raising an exception.
+    (:issue:`5881`)
+
+-   :func:`scrapy.utils.python.get_func_args` now supports more types of
+    callables. (:issue:`5872`, :issue:`5885`)
+
 -   Fixed an error breaking user handling of send failures in
     :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
 

From 636559f1cc652e839ef42522e7168e3ff9d77921 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 4 May 2023 17:55:07 +0400
Subject: [PATCH 25/64] Add newer changes.

---
 docs/news.rst | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 5f189760d..cbbb376e5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -12,7 +12,8 @@ Highlights:
 
 -   Per-domain download settings.
 -   Compatibility with new cryptography_ and new parsel_.
--   TBD
+-   JMESPath selectors from the new parsel_.
+-   Bug fixes.
 
 New features
 ~~~~~~~~~~~~
@@ -22,6 +23,12 @@ New features
     :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
     via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
 
+-   Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors
+    available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`)
+
+-   Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed`
+    signals. (:issue:`5876`)
+
 -   Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a
     curl command from a :class:`~scrapy.Request` object. (:issue:`5892`)
 
@@ -31,6 +38,8 @@ New features
 Bug fixes
 ~~~~~~~~~
 
+-   Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`)
+
 -   Fixed an error when using feed postprocessing with S3 storage.
     (:issue:`5500`, :issue:`5581`)
 
@@ -65,6 +74,9 @@ Bug fixes
 -   :func:`scrapy.utils.python.get_func_args` now supports more types of
     callables. (:issue:`5872`, :issue:`5885`)
 
+-   Fixed an error when processing non-UTF8 values of ``Content-Type`` headers.
+    (:issue:`5914`, :issue:`5917`)
+
 -   Fixed an error breaking user handling of send failures in
     :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
 
@@ -89,7 +101,7 @@ Quality assurance
 -   Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`)
 
 -   Tests for most of the examples in the docs are now run as a part of CI,
-    found problems were fixed. (:issue:`5816`, :issue:`5826`)
+    found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`)
 
 -   Removed usage of deprecated Python classes. (:issue:`5849`)
 
@@ -99,9 +111,10 @@ Quality assurance
     test. (:issue:`5855`, :issue:`5898`)
 
 -   Updated docstrings to match output produced by parsel_ 1.8.1 so that they
-    don't cause test failures. (:issue:`5902`)
+    don't cause test failures. (:issue:`5902`, :issue:`5919`)
 
--   Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`)
+-   Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`,
+    :issue:`5908`)
 
 .. _blacken-docs: https://github.com/adamchainz/blacken-docs
 

From caa66fa15ae5d353434e5bb1d0c1cc2bdf857b6c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 8 May 2023 13:27:01 +0400
Subject: [PATCH 26/64] Mention deprecating _FeedSlot.

---
 docs/news.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index cbbb376e5..4e198c893 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -15,6 +15,13 @@ Highlights:
 -   JMESPath selectors from the new parsel_.
 -   Bug fixes.
 
+Deprecations
+~~~~~~~~~~~~
+
+-   :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to
+    :class:`scrapy.extensions.feedexport.FeedSlot` and the old name is
+    deprecated. (:issue:`5876`)
+
 New features
 ~~~~~~~~~~~~
 

From 52c072640aa61884de05214cb1bdda07c2a87bef Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 8 May 2023 14:30:06 +0400
Subject: [PATCH 27/64] =?UTF-8?q?Bump=20version:=202.8.0=20=E2=86=92=202.9?=
 =?UTF-8?q?.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 docs/news.rst    | 2 +-
 scrapy/VERSION   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 4cfba674d..a00b7cfb3 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.8.0
+current_version = 2.9.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/docs/news.rst b/docs/news.rst
index 4e198c893..c7ad11862 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-2.9.0:
 
-Scrapy 2.9.0 (YYYY-MM-DD)
+Scrapy 2.9.0 (2023-05-08)
 -------------------------
 
 Highlights:
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 834f26295..c8e38b614 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.8.0
+2.9.0

From c327a92e971411e50e49dded772a73b293f9f0a9 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Tue, 9 May 2023 18:04:18 +0500
Subject: [PATCH 28/64] add additional requests examples.

---
 docs/topics/asyncio.rst | 47 +++++++++++++++++++++++++++++++++++++++++
 scrapy/utils/defer.py   | 10 +++++----
 2 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 7713b1af1..7aa83f505 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -98,6 +98,53 @@ Futures. Scrapy provides two helpers for this:
          into your own code.
 
 
+Async additional requests
+=====================
+
+The spider below shows a single use-case of scraping page and gathering price from a separate url::
+
+
+    class SingleRequestSpider(scrapy.Spider):
+        name = "single"
+        start_urls = ["https://example.org/product"]
+
+        async def parse(self, response, **kwargs):
+            additional_request = scrapy.Request('https://example.org/price')
+            deferred = self.crawler.engine.download(additional_request)
+            additional_response = await maybe_deferred_to_future(deferred)
+            yield {
+                'h1': response.css('h1').get(),
+                'price': additional_response.css('#price').get(),
+            }
+
+
+Spider with gathering batch requests::
+
+    class BatchRequestsSpider(scrapy.Spider):
+        name = "batch"
+        start_urls = ["https://example.com/product"]
+
+        async def parse(self, response, **kwargs):
+            additional_requests = [
+                scrapy.Request("https://example.com/price1"),
+                scrapy.Request("https://example.com/price2"),
+            ]
+            coroutines = []
+            for r in additional_requests:
+                deffered = self.crawler.engine.download(r)
+                coroutines.append(maybe_deferred_to_future(deffered))
+
+            responses = await asyncio.gather(
+                *coroutines, return_exceptions=True
+            )
+            yield {
+                'h1': response.css('h1::text').get(),
+                'price': responses[0].css('.price_color::text').get(),
+                'price2': responses[1].css('.price_color::text').get(),
+            }
+
+
+
 .. _enforce-asyncio-requirement:
 
 Enforcing asyncio as a requirement
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d25ebbdf4..a46274fef 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future:
         class MySpider(Spider):
             ...
             async def parse(self, response):
-                d = treq.get('https://example.com/additional')
-                additional_response = await deferred_to_future(d)
+                additional_request = scrapy.Request('https://example.org/price')
+                deferred = self.crawler.engine.download(additional_request)
+                additional_response = await deferred_to_future(deferred)
     """
     return d.asFuture(_get_asyncio_event_loop())
 
@@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
         class MySpider(Spider):
             ...
             async def parse(self, response):
-                d = treq.get('https://example.com/additional')
-                extra_response = await maybe_deferred_to_future(d)
+                additional_request = scrapy.Request('https://example.org/price')
+                deferred = self.crawler.engine.download(additional_request)
+                additional_response = await maybe_deferred_to_future(deferred)
     """
     if not is_asyncio_reactor_installed():
         return d

From a75231a1ecd9a08b31ea3c1d5b59e457ac85ccf2 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Tue, 9 May 2023 18:57:43 +0500
Subject: [PATCH 29/64] fix underline.

---
 docs/topics/asyncio.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 7aa83f505..d46527cfc 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this:
 
 
 Async additional requests
-=====================
+=========================
 
 The spider below shows a single use-case of scraping page and gathering price from a separate url::
 

From d32c6782347c97086e804da0da01f45622743198 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Tue, 9 May 2023 19:02:34 +0500
Subject: [PATCH 30/64] Update description.

---
 docs/topics/asyncio.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index d46527cfc..d439e0ab8 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -101,7 +101,7 @@ Futures. Scrapy provides two helpers for this:
 Async additional requests
 =========================
 
-The spider below shows a single use-case of scraping page and gathering price from a separate url::
+The spider below shows a single use-case of scraping a page and gathering a price from a separate URL::
 
 
     class SingleRequestSpider(scrapy.Spider):
@@ -118,7 +118,7 @@ The spider below shows a single use-case of scraping page and gathering price fr
             }
 
 
-Spider with gathering batch requests::
+The spider gathering batch requests::
 
     class BatchRequestsSpider(scrapy.Spider):
         name = "batch"

From 99b0ece165ff27b70a6d7375a86bcc67da111df7 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Tue, 9 May 2023 20:27:46 +0500
Subject: [PATCH 31/64] remove extra line.

---
 docs/topics/asyncio.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index d439e0ab8..0dab0ac5e 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -144,7 +144,6 @@ The spider gathering batch requests::
             }
 
 
-
 .. _enforce-asyncio-requirement:
 
 Enforcing asyncio as a requirement

From b1f4017788877ba2139f8621ecb5e821c62c111d Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Wed, 10 May 2023 15:34:58 +0500
Subject: [PATCH 32/64] Refactor batch sample.

---
 docs/topics/asyncio.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 0dab0ac5e..dc83148f5 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -126,8 +126,8 @@ The spider gathering batch requests::
 
         async def parse(self, response, **kwargs):
             additional_requests = [
-                scrapy.Request("https://example.com/price1"),
-                scrapy.Request("https://example.com/price2"),
+                scrapy.Request("https://example.com/price"),
+                scrapy.Request("https://example.com/color"),
             ]
             coroutines = []
             for r in additional_requests:
@@ -139,8 +139,8 @@ The spider gathering batch requests::
             )
             yield {
                 'h1': response.css('h1::text').get(),
-                'price': responses[0].css('.price_color::text').get(),
-                'price2': responses[1].css('.price_color::text').get(),
+                'price': responses[0].css('.price::text').get(),
+                'color': responses[1].css('color::text').get(),
             }
 
 

From 87d10161cd413353eba2abf8ebdc2a8656927c43 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Wed, 10 May 2023 15:35:48 +0500
Subject: [PATCH 33/64] Add selector as class.

---
 docs/topics/asyncio.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index dc83148f5..f00ba0ff8 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -140,7 +140,7 @@ The spider gathering batch requests::
             yield {
                 'h1': response.css('h1::text').get(),
                 'price': responses[0].css('.price::text').get(),
-                'color': responses[1].css('color::text').get(),
+                'color': responses[1].css('.color::text').get(),
             }
 
 

From 57f3140daaa0166f924fcca42d3f3d3ef178bf92 Mon Sep 17 00:00:00 2001
From: Bulat Khabibullin <xb.bolo@gmail.com>
Date: Wed, 10 May 2023 18:31:54 +0500
Subject: [PATCH 34/64] Update docs/topics/asyncio.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/asyncio.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index f00ba0ff8..f9efef108 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -101,8 +101,10 @@ Futures. Scrapy provides two helpers for this:
 Async additional requests
 =========================
 
-The spider below shows a single use-case of scraping a page and gathering a price from a separate URL::
+The spider below shows how to send a request and await its response all from 
+within a spider callback:
 
+.. code-block:: python
 
     class SingleRequestSpider(scrapy.Spider):
         name = "single"

From 26374e21f81eb53f5a21e1cc68a65e54fbb62cb6 Mon Sep 17 00:00:00 2001
From: Bulat Khabibullin <xb.bolo@gmail.com>
Date: Wed, 10 May 2023 18:32:36 +0500
Subject: [PATCH 35/64] Update docs/topics/asyncio.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/asyncio.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index f9efef108..7fe78585a 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -119,8 +119,9 @@ within a spider callback:
                 'price': additional_response.css('#price').get(),
             }
 
+You can also send multiple requests in parallel:
 
-The spider gathering batch requests::
+.. code-block:: python
 
     class BatchRequestsSpider(scrapy.Spider):
         name = "batch"

From 85103b493289011161731e4fafec4320cd85e0af Mon Sep 17 00:00:00 2001
From: Bulat Khabibullin <xb.bolo@gmail.com>
Date: Thu, 11 May 2023 12:53:43 +0500
Subject: [PATCH 36/64] add proper example
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/asyncio.rst | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 7fe78585a..eeec76157 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -123,6 +123,8 @@ You can also send multiple requests in parallel:
 
 .. code-block:: python
 
+    from twisted.internet.defer import DeferredList
+
     class BatchRequestsSpider(scrapy.Spider):
         name = "batch"
         start_urls = ["https://example.com/product"]
@@ -132,14 +134,11 @@ You can also send multiple requests in parallel:
                 scrapy.Request("https://example.com/price"),
                 scrapy.Request("https://example.com/color"),
             ]
-            coroutines = []
+            deferreds = []
             for r in additional_requests:
-                deffered = self.crawler.engine.download(r)
-                coroutines.append(maybe_deferred_to_future(deffered))
-
-            responses = await asyncio.gather(
-                *coroutines, return_exceptions=True
-            )
+                deferred = self.crawler.engine.download(r)
+                deferreds.append(deferred)
+            responses = await maybe_deferred_to_future(DeferredList(deferreds))
             yield {
                 'h1': response.css('h1::text').get(),
                 'price': responses[0].css('.price::text').get(),

From 6194db133518b07b92bfdae35332c69e95f5c415 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Thu, 11 May 2023 12:54:01 +0500
Subject: [PATCH 37/64] Update title.

---
 docs/topics/asyncio.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 7fe78585a..1b670aaab 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -98,7 +98,7 @@ Futures. Scrapy provides two helpers for this:
          into your own code.
 
 
-Async additional requests
+Inline requests
 =========================
 
 The spider below shows how to send a request and await its response all from 

From fc2d1b217130ebf605636049ea773196a50303a0 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Thu, 11 May 2023 12:56:29 +0500
Subject: [PATCH 38/64] make example reachable.

---
 docs/topics/asyncio.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 12bf548df..fcf44c0cb 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -125,8 +125,8 @@ You can also send multiple requests in parallel:
 
     from twisted.internet.defer import DeferredList
 
-    class BatchRequestsSpider(scrapy.Spider):
-        name = "batch"
+    class MultipleRequestsSpider(scrapy.Spider):
+        name = "multiple"
         start_urls = ["https://example.com/product"]
 
         async def parse(self, response, **kwargs):
@@ -141,8 +141,8 @@ You can also send multiple requests in parallel:
             responses = await maybe_deferred_to_future(DeferredList(deferreds))
             yield {
                 'h1': response.css('h1::text').get(),
-                'price': responses[0].css('.price::text').get(),
-                'color': responses[1].css('.color::text').get(),
+                'price': responses[0][1].css('.price::text').get(),
+                'price2': responses[1][1].css('.color::text').get(),
             }
 
 

From b62c1263de4b026e8529416d5dede1795d47f7ad Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Thu, 11 May 2023 13:15:30 +0500
Subject: [PATCH 39/64] add import to the example.

---
 docs/topics/asyncio.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index fcf44c0cb..2ad784335 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -106,6 +106,8 @@ within a spider callback:
 
 .. code-block:: python
 
+    from scrapy.utils.defer import maybe_deferred_to_future
+
     class SingleRequestSpider(scrapy.Spider):
         name = "single"
         start_urls = ["https://example.org/product"]

From 4878cc7ef04ae40279d80fec5d5234d17569ce11 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Thu, 11 May 2023 13:19:40 +0500
Subject: [PATCH 40/64] Add proper imports.

---
 docs/topics/asyncio.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 2ad784335..a3f45a84b 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -125,7 +125,7 @@ You can also send multiple requests in parallel:
 
 .. code-block:: python
 
-    from twisted.internet.defer import DeferredList
+    from scrapy.utils.defer import DeferredList
 
     class MultipleRequestsSpider(scrapy.Spider):
         name = "multiple"

From 8de2064ba33d6e0b8e0a22a6b5f6928a35eb44b7 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Thu, 11 May 2023 13:22:33 +0500
Subject: [PATCH 41/64] add import.

---
 docs/topics/asyncio.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index a3f45a84b..5e0063be0 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -125,7 +125,7 @@ You can also send multiple requests in parallel:
 
 .. code-block:: python
 
-    from scrapy.utils.defer import DeferredList
+    from scrapy.utils.defer import DeferredList, maybe_deferred_to_future
 
     class MultipleRequestsSpider(scrapy.Spider):
         name = "multiple"

From 5adada5d19e9e275462330b070b746305115112e Mon Sep 17 00:00:00 2001
From: isabela_catanante <isabela.catanante@zyte.com>
Date: Fri, 12 May 2023 12:55:24 +0200
Subject: [PATCH 42/64] Improve the overwrite feed option documentation

---
 docs/topics/feed-exports.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index eef0bb5ca..b4ac93b1d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive
 mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
+storage backend is: ``True``.
+
+.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
+     previous version of your data.
+
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 
@@ -209,6 +215,12 @@ You can also define a custom ACL and custom endpoint for exported feeds using th
 -   :setting:`FEED_STORAGE_S3_ACL`
 -   :setting:`AWS_ENDPOINT_URL`
 
+The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
+storage backend is: ``True``.
+
+.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
+     previous version of your data.
+
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 
@@ -236,6 +248,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
 -   :setting:`FEED_STORAGE_GCS_ACL`
 -   :setting:`GCS_PROJECT_ID`
 
+The default value for the ``overwrite`` key in the :setting:`FEEDS` for this 
+storage backend is: ``True``.
+
+.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
+     previous version of your data.
+
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
@@ -488,6 +506,8 @@ as a fallback value if that key is not provided for a specific feed definition:
     -   :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
         <https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
 
+    -   :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported)
+
     -   :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
 
     .. versionadded:: 2.4.0

From e4cf8fc121fc89d70949a9159bfe67cbd0429e71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 15 May 2023 18:51:58 +0200
Subject: [PATCH 43/64] Update asyncio.rst

---
 docs/topics/asyncio.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 5e0063be0..efb93c844 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this:
 
 
 Inline requests
-=========================
+===============
 
 The spider below shows how to send a request and await its response all from 
 within a spider callback:

From d362699fa3855c7fd6e11204ccd8668128e38a5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 16 May 2023 13:39:02 +0200
Subject: [PATCH 44/64] Move inline request examples to the coroutines
 documentation

---
 docs/topics/asyncio.rst    | 50 ---------------------------------
 docs/topics/coroutines.rst | 57 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 50 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index efb93c844..7713b1af1 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this:
          into your own code.
 
 
-Inline requests
-===============
-
-The spider below shows how to send a request and await its response all from 
-within a spider callback:
-
-.. code-block:: python
-
-    from scrapy.utils.defer import maybe_deferred_to_future
-
-    class SingleRequestSpider(scrapy.Spider):
-        name = "single"
-        start_urls = ["https://example.org/product"]
-
-        async def parse(self, response, **kwargs):
-            additional_request = scrapy.Request('https://example.org/price')
-            deferred = self.crawler.engine.download(additional_request)
-            additional_response = await maybe_deferred_to_future(deferred)
-            yield {
-                'h1': response.css('h1').get(),
-                'price': additional_response.css('#price').get(),
-            }
-
-You can also send multiple requests in parallel:
-
-.. code-block:: python
-
-    from scrapy.utils.defer import DeferredList, maybe_deferred_to_future
-
-    class MultipleRequestsSpider(scrapy.Spider):
-        name = "multiple"
-        start_urls = ["https://example.com/product"]
-
-        async def parse(self, response, **kwargs):
-            additional_requests = [
-                scrapy.Request("https://example.com/price"),
-                scrapy.Request("https://example.com/color"),
-            ]
-            deferreds = []
-            for r in additional_requests:
-                deferred = self.crawler.engine.download(r)
-                deferreds.append(deferred)
-            responses = await maybe_deferred_to_future(DeferredList(deferreds))
-            yield {
-                'h1': response.css('h1::text').get(),
-                'price': responses[0][1].css('.price::text').get(),
-                'price2': responses[1][1].css('.color::text').get(),
-            }
-
-
 .. _enforce-asyncio-requirement:
 
 Enforcing asyncio as a requirement
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 3916bd295..a65bab3ca 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -134,6 +134,63 @@ Common use cases for asynchronous code include:
 .. _aio-libs: https://github.com/aio-libs
 
 
+.. _inline-requests:
+
+Inline requests
+===============
+
+The spider below shows how to send a request and await its response all from
+within a spider callback:
+
+.. code-block:: python
+
+    from scrapy import Spider, Request
+    from scrapy.utils.defer import maybe_deferred_to_future
+
+
+    class SingleRequestSpider(Spider):
+        name = "single"
+        start_urls = ["https://example.org/product"]
+
+        async def parse(self, response, **kwargs):
+            additional_request = Request("https://example.org/price")
+            deferred = self.crawler.engine.download(additional_request)
+            additional_response = await maybe_deferred_to_future(deferred)
+            yield {
+                "h1": response.css("h1").get(),
+                "price": additional_response.css("#price").get(),
+            }
+
+You can also send multiple requests in parallel:
+
+.. code-block:: python
+
+    from scrapy import Spider, Request
+    from scrapy.utils.defer import maybe_deferred_to_future
+    from twisted.internet.defer import DeferredList
+
+
+    class MultipleRequestsSpider(Spider):
+        name = "multiple"
+        start_urls = ["https://example.com/product"]
+
+        async def parse(self, response, **kwargs):
+            additional_requests = [
+                Request("https://example.com/price"),
+                Request("https://example.com/color"),
+            ]
+            deferreds = []
+            for r in additional_requests:
+                deferred = self.crawler.engine.download(r)
+                deferreds.append(deferred)
+            responses = await maybe_deferred_to_future(DeferredList(deferreds))
+            yield {
+                "h1": response.css("h1::text").get(),
+                "price": responses[0][1].css(".price::text").get(),
+                "price2": responses[1][1].css(".color::text").get(),
+            }
+
+
 .. _sync-async-spider-middleware:
 
 Mixing synchronous and asynchronous spider middlewares

From 49839d6071832aab23093c34fa6ceb961fcdf9d0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 4 Jun 2023 19:59:58 +0400
Subject: [PATCH 45/64] Don't rely on get_testenv() for running mockserver.

---
 tests/mockserver.py         | 19 +++++++++++++++----
 tests/test_crawler.py       | 11 ++++++++---
 tests/test_proxy_connect.py |  3 ---
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index eb4f8334e..647b0682e 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,11 +1,13 @@
 import argparse
 import json
+import os
 import random
 import sys
 from pathlib import Path
 from shutil import rmtree
 from subprocess import PIPE, Popen
 from tempfile import mkdtemp
+from typing import Dict
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
@@ -20,7 +22,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.test import get_testenv
 
 
 def getarg(request, name, default=None, type=None):
@@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None):
     return default
 
 
+def get_mockserver_env() -> Dict[str, str]:
+    """Return a OS environment dict suitable to run mockserver processes."""
+
+    tests_path = Path(__file__).parent.parent
+    pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "")
+    env = os.environ.copy()
+    env["PYTHONPATH"] = pythonpath
+    return env
+
+
 # most of the following resources are copied from twisted.web.test.test_webclient
 class ForeverTakingResource(resource.Resource):
     """
@@ -264,7 +275,7 @@ class MockServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         http_address = self.proc.stdout.readline().strip().decode("ascii")
         https_address = self.proc.stdout.readline().strip().decode("ascii")
@@ -308,7 +319,7 @@ class MockDNSServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"],
             stdout=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         self.host = "127.0.0.1"
         self.port = int(
@@ -331,7 +342,7 @@ class MockFTPServer:
         self.proc = Popen(
             [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)],
             stderr=PIPE,
-            env=get_testenv(),
+            env=get_mockserver_env(),
         )
         for line in self.proc.stderr:
             if b"starting FTP server" in line:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 706bfbaa9..ecb9c9b62 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,5 @@
 import logging
+import os
 import platform
 import subprocess
 import sys
@@ -23,8 +24,8 @@ from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.misc import load_object
 from scrapy.utils.spider import DefaultSpider
-from scrapy.utils.test import get_crawler, get_testenv
-from tests.mockserver import MockServer
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer, get_mockserver_env
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -289,12 +290,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
 class ScriptRunnerMixin:
     script_dir: Path
+    cwd = os.getcwd()
 
     def run_script(self, script_name: str, *script_args):
         script_path = self.script_dir / script_name
         args = [sys.executable, str(script_path)] + list(script_args)
         p = subprocess.Popen(
-            args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            args,
+            env=get_mockserver_env(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         stdout, stderr = p.communicate()
         return stderr.decode("utf-8")
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index c05f4da91..dc0a82086 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -21,8 +21,6 @@ class MitmProxy:
     auth_pass = "scrapy"
 
     def start(self):
-        from scrapy.utils.test import get_testenv
-
         script = """
 import sys
 from mitmproxy.tools.main import mitmdump
@@ -46,7 +44,6 @@ sys.exit(mitmdump())
                 "--ssl-insecure",
             ],
             stdout=PIPE,
-            env=get_testenv(),
         )
         line = self.proc.stdout.readline().decode("utf-8")
         host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)

From 493ea435384d4b5891129cbcf61d07b09a00ce7f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 12 Jun 2023 21:50:29 +0400
Subject: [PATCH 46/64] Improve finding tests.test_cmdline.settings.

---
 tests/test_cmdline/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 15833cd19..25ded143c 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,4 +1,5 @@
 import json
+import os
 import pstats
 import shutil
 import sys
@@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv
 class CmdlineTest(unittest.TestCase):
     def setUp(self):
         self.env = get_testenv()
+        tests_path = Path(__file__).parent.parent
+        self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
         self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
 
     def _execute(self, *new_args, **kwargs):

From 0adbd210acc9d1af75e42df2b17ee28e3fecdfc8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 13 Jun 2023 19:34:26 +0400
Subject: [PATCH 47/64] Fix extra-deps-pinned tests.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 8a6693a49..5f8bf85f2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -139,6 +139,7 @@ deps =
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
+commands = {[pinned]commands}
 
 [testenv:asyncio]
 commands =

From 7cfdca8f9b1b1f16aacae958f7a5c8824df056e7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 14 Jun 2023 18:23:29 +0400
Subject: [PATCH 48/64] Actually run test_set_asyncio_event_loop().

---
 tests/test_utils_asyncio.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 01d0ee043..65e352053 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -4,6 +4,7 @@ from unittest import TestCase
 
 from pytest import mark
 
+from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.reactor import (
     install_reactor,
     is_asyncio_reactor_installed,
@@ -29,6 +30,8 @@ class AsyncioTest(TestCase):
 
         assert original_reactor == reactor
 
+    @mark.only_asyncio()
+    @deferred_f_from_coro_f
     async def test_set_asyncio_event_loop(self):
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
-        assert set_asyncio_event_loop() is asyncio.get_running_loop()
+        assert set_asyncio_event_loop(None) is asyncio.get_running_loop()

From 777a6ea4128cf00d53fa71dc48385bf036cb64fc Mon Sep 17 00:00:00 2001
From: Serhii A <aserhii@protonmail.com>
Date: Fri, 16 Jun 2023 16:46:06 +0300
Subject: [PATCH 49/64] Make the retry middleware exception list configurable
 (#5929)

---
 docs/topics/downloader-middleware.rst    | 32 ++++++++++++
 scrapy/downloadermiddlewares/retry.py    | 63 ++++++++++++------------
 scrapy/settings/default_settings.py      | 15 ++++++
 tests/test_downloadermiddleware_retry.py | 39 +++++++++++++--
 4 files changed, 113 insertions(+), 36 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 7665a901a..a8e5b23bf 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -915,6 +915,7 @@ settings (see the settings documentation for more info):
 * :setting:`RETRY_ENABLED`
 * :setting:`RETRY_TIMES`
 * :setting:`RETRY_HTTP_CODES`
+* :setting:`RETRY_EXCEPTIONS`
 
 .. reqmeta:: dont_retry
 
@@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
 it is a common code used to indicate server overload. It is not included by
 default because HTTP specs say so.
 
+.. setting:: RETRY_EXCEPTIONS
+
+RETRY_EXCEPTIONS
+^^^^^^^^^^^^^^^^
+
+Default::
+
+    [
+        'twisted.internet.defer.TimeoutError',
+        'twisted.internet.error.TimeoutError',
+        'twisted.internet.error.DNSLookupError',
+        'twisted.internet.error.ConnectionRefusedError',
+        'twisted.internet.error.ConnectionDone',
+        'twisted.internet.error.ConnectError',
+        'twisted.internet.error.ConnectionLost',
+        'twisted.internet.error.TCPTimedOutError',
+        'twisted.web.client.ResponseFailed',
+        IOError,
+        'scrapy.core.downloader.handlers.http11.TunnelError',
+    ]
+
+List of exceptions to retry.
+
+Each list entry may be an exception type or its import path as a string.
+
+An exception will not be caught when the exception type is not in
+:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
+has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
+exception propagation, see
+:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
+
 .. setting:: RETRY_PRIORITY_ADJUST
 
 RETRY_PRIORITY_ADJUST
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 081642a4b..50cbc3111 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+import warnings
 from logging import Logger, getLogger
 from typing import Optional, Union
 
-from twisted.internet import defer
-from twisted.internet.error import (
-    ConnectError,
-    ConnectionDone,
-    ConnectionLost,
-    ConnectionRefusedError,
-    DNSLookupError,
-    TCPTimedOutError,
-    TimeoutError,
-)
-from twisted.web.client import ResponseFailed
-
-from scrapy.core.downloader.handlers.http11 import TunnelError
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http.request import Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
 
 retry_logger = getLogger(__name__)
 
 
+class BackwardsCompatibilityMetaclass(type):
+    @property
+    def EXCEPTIONS_TO_RETRY(cls):
+        warnings.warn(
+            "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
+            "Use the RETRY_EXCEPTIONS setting instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return tuple(
+            load_object(x) if isinstance(x, str) else x
+            for x in Settings().getlist("RETRY_EXCEPTIONS")
+        )
+
+
 def get_retry_request(
     request: Request,
     *,
@@ -121,23 +126,7 @@ def get_retry_request(
     return None
 
 
-class RetryMiddleware:
-    # IOError is raised by the HttpCompression middleware when trying to
-    # decompress an empty response
-    EXCEPTIONS_TO_RETRY = (
-        defer.TimeoutError,
-        TimeoutError,
-        DNSLookupError,
-        ConnectionRefusedError,
-        ConnectionDone,
-        ConnectError,
-        ConnectionLost,
-        TCPTimedOutError,
-        ResponseFailed,
-        IOError,
-        TunnelError,
-    )
-
+class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
     def __init__(self, settings):
         if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
@@ -147,6 +136,16 @@ class RetryMiddleware:
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
+        if not hasattr(
+            self, "EXCEPTIONS_TO_RETRY"
+        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+            self.exceptions_to_retry = tuple(
+                load_object(x) if isinstance(x, str) else x
+                for x in settings.getlist("RETRY_EXCEPTIONS")
+            )
+        else:
+            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
+
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
@@ -160,7 +159,7 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
+        if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
             "dont_retry", False
         ):
             return self._retry(request, exception, spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 260ec1701..89837b4ab 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -258,6 +258,21 @@ RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
+RETRY_EXCEPTIONS = [
+    "twisted.internet.defer.TimeoutError",
+    "twisted.internet.error.TimeoutError",
+    "twisted.internet.error.DNSLookupError",
+    "twisted.internet.error.ConnectionRefusedError",
+    "twisted.internet.error.ConnectionDone",
+    "twisted.internet.error.ConnectError",
+    "twisted.internet.error.ConnectionLost",
+    "twisted.internet.error.TCPTimedOutError",
+    "twisted.web.client.ResponseFailed",
+    # IOError is raised by the HttpCompression middleware when trying to
+    # decompress an empty response
+    IOError,
+    "scrapy.core.downloader.handlers.http11.TunnelError",
+]
 
 ROBOTSTXT_OBEY = False
 ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 63bd61848..97ae1e29a 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,5 +1,6 @@
 import logging
 import unittest
+import warnings
 
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
+from scrapy.settings.default_settings import RETRY_EXCEPTIONS
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
@@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase):
             == 2
         )
 
-    def _test_retry_exception(self, req, exception):
+    def test_exception_to_retry_added(self):
+        exc = ValueError
+        settings_dict = {
+            "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc],
+        }
+        crawler = get_crawler(Spider, settings_dict=settings_dict)
+        mw = RetryMiddleware.from_crawler(crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        self._test_retry_exception(req, exc("foo"), mw)
+
+    def test_exception_to_retry_customMiddleware(self):
+        exc = ValueError
+
+        with warnings.catch_warnings(record=True) as warns:
+
+            class MyRetryMiddleware(RetryMiddleware):
+                EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
+
+            self.assertEqual(len(warns), 1)
+
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+
+    def _test_retry_exception(self, req, exception, mw=None):
+        if mw is None:
+            mw = self.mw
+
         # first retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
         # second retry
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 2)
 
         # discard it
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = mw.process_exception(req, exception, self.spider)
         self.assertEqual(req, None)
 
 

From 2122278d4bb7177a550bc0b04dd96d1fc1fad1a0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 18 Jun 2023 18:37:50 +0400
Subject: [PATCH 50/64] Drop Python 3.7 support.

---
 .github/workflows/tests-macos.yml   |  2 +-
 .github/workflows/tests-ubuntu.yml  | 13 +++++--------
 .github/workflows/tests-windows.yml |  5 +----
 README.rst                          |  2 +-
 docs/contributing.rst               | 12 ++++++------
 docs/intro/install.rst              |  2 +-
 scrapy/__init__.py                  |  4 ++--
 setup.py                            |  3 +--
 tox.ini                             |  8 +++-----
 9 files changed, 21 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 174d245ca..3044a1af3 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 96b26a1f8..39e3b0af7 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -8,9 +8,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.8
-          env:
-            TOXENV: py
         - python-version: 3.9
           env:
             TOXENV: py
@@ -28,19 +25,19 @@ jobs:
             TOXENV: pypy3
 
         # pinned deps
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: asyncio-pinned
-        - python-version: pypy3.7
+        - python-version: pypy3.8
           env:
             TOXENV: pypy3-pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: extra-deps-pinned
-        - python-version: 3.7.13
+        - python-version: 3.8.17
           env:
             TOXENV: botocore-pinned
 
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index f60c48841..5bcf74d5e 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -8,12 +8,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.7
-          env:
-            TOXENV: windows-pinned
         - python-version: 3.8
           env:
-            TOXENV: py
+            TOXENV: windows-pinned
         - python-version: 3.9
           env:
             TOXENV: py
diff --git a/README.rst b/README.rst
index 970bf2c35..1918850d6 100644
--- a/README.rst
+++ b/README.rst
@@ -58,7 +58,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.7+
+* Python 3.8+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/contributing.rst b/docs/contributing.rst
index eef92e148..2b3249601 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
 To run the tests on a specific :doc:`tox <tox:index>` environment, use
 ``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.7 use::
+the tests with Python 3.10 use::
 
-    tox -e py37
+    tox -e py310
 
 You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
-    tox -e py37,py38 -p auto
+    tox -e py39,py310 -p auto
 
 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
     tox -- scrapy tests -x  # stop after first failure
 
 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
 
-    tox -e py37 -- scrapy tests -n auto
+    tox -e py310 -- scrapy tests -n auto
 
 To see coverage report install :doc:`coverage <coverage:index>`
 (``pip install coverage``) and run:
@@ -322,4 +322,4 @@ And their unit-tests are in::
 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
 .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
-.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
\ No newline at end of file
+.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 2c2079f68..c90c1d2bf 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,7 +9,7 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.7+, either the CPython implementation (default) or
+Scrapy requires Python 3.8+, either the CPython implementation (default) or
 the PyPy implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index a757a9290..cc0e539c4 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 7):
-    print(f"Scrapy {__version__} requires Python 3.7+")
+if sys.version_info < (3, 8):
+    print(f"Scrapy {__version__} requires Python 3.8+")
     sys.exit(1)
 
 
diff --git a/setup.py b/setup.py
index c6bcf2439..f1cd4c5e2 100644
--- a/setup.py
+++ b/setup.py
@@ -80,7 +80,6 @@ setup(
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
@@ -91,7 +90,7 @@ setup(
         "Topic :: Software Development :: Libraries :: Application Frameworks",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tox.ini b/tox.ini
index 5f8bf85f2..d5b6118f5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,8 +16,6 @@ deps =
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
     mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
-    # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
-    markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -96,7 +94,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
 
 [testenv:pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     PyDispatcher==2.0.5
@@ -129,7 +127,7 @@ deps =
     Twisted[http2]
 
 [testenv:extra-deps-pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     boto3==1.20.0
@@ -211,7 +209,7 @@ commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
 
 [testenv:botocore-pinned]
-basepython = python3.7
+basepython = python3.8
 deps =
     {[pinned]deps}
     botocore==1.4.87

From 1b2c9a3e0ae9766623a06ce7e739a3dfda399159 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 18 Jun 2023 18:38:56 +0400
Subject: [PATCH 51/64] Bump isort and flake8 versions.

---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index faf8808f2..31e9ed1ad 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ repos:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4  # 6.0.0 drops Python 3.7 support
+  rev: 6.0.0
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
@@ -13,7 +13,7 @@ repos:
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.11.5  # 5.12 drops Python 3.7 support
+  rev: 5.12.0
   hooks:
   - id: isort
 - repo: https://github.com/adamchainz/blacken-docs

From 075b89eab5e201246a5bddc4a6ce9c7921de082b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 18 Jun 2023 19:08:41 +0400
Subject: [PATCH 52/64] Bump lxml and cryptography to versions with 3.8 wheels
 available.

---
 setup.py | 4 ++--
 tox.ini  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index f1cd4c5e2..ccfe20ae5 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     "Twisted>=18.9.0",
-    "cryptography>=3.4.6",
+    "cryptography>=36.0.0",
     "cssselect>=0.9.1",
     "itemloaders>=1.0.1",
     "parsel>=1.5.0",
@@ -34,7 +34,7 @@ install_requires = [
     "setuptools",
     "packaging",
     "tldextract",
-    "lxml>=4.3.0",
+    "lxml>=4.4.1",
 ]
 extras_require = {}
 cpython_dependencies = [
diff --git a/tox.ini b/tox.ini
index d5b6118f5..ec3a59366 100644
--- a/tox.ini
+++ b/tox.ini
@@ -69,7 +69,7 @@ commands =
 
 [pinned]
 deps =
-    cryptography==3.4.6
+    cryptography==36.0.0
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
@@ -81,7 +81,7 @@ deps =
     Twisted[http2]==18.9.0
     w3lib==1.17.0
     zope.interface==5.1.0
-    lxml==4.3.0
+    lxml==4.4.1
     -rtests/requirements.txt
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies

From 0097b4c0bb4de6e651e8b9d064aae140e11698d5 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 16:40:38 -0300
Subject: [PATCH 53/64] cleanup: Remove `pkg_resources` usage

---
 docs/topics/components.rst | 2 +-
 scrapy/cmdline.py          | 5 ++---
 setup.py                   | 2 +-
 tests/test_crawler.py      | 2 +-
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index 1ed55f000..478dd9647 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use
 
 .. code-block:: python
 
-    from pkg_resources import parse_version
+    from packaging.version import parse as parse_version
 
     import scrapy
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 730e55350..cfa771104 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -3,8 +3,7 @@ import cProfile
 import inspect
 import os
 import sys
-
-import pkg_resources
+from importlib.metadata import entry_points
 
 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@@ -49,7 +48,7 @@ def _get_commands_from_module(module, inproject):
 
 def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
-    for entry_point in pkg_resources.iter_entry_points(group):
+    for entry_point in entry_points(group):
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()
diff --git a/setup.py b/setup.py
index ccfe20ae5..f918db09e 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ecb9c9b62..d54a2cb7e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -6,7 +6,7 @@ import sys
 import warnings
 from pathlib import Path
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 from pytest import mark, raises
 from twisted import version as twisted_version
 from twisted.internet import defer

From 6afb31b82b5a0a5d2f37962c250fbf34c21d8580 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 16:48:07 -0300
Subject: [PATCH 54/64] chore: Add `packaging` to tests deps

---
 tests/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index 618949795..72350b216 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -14,3 +14,4 @@ brotli  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
+packaging
\ No newline at end of file

From 6e1af20ac4dd537a4643df5c022f948cf07d05ec Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 17:00:01 -0300
Subject: [PATCH 55/64] fix: add `build-system`

---
 tox.ini | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tox.ini b/tox.ini
index ec3a59366..79d692599 100644
--- a/tox.ini
+++ b/tox.ini
@@ -218,3 +218,6 @@ setenv =
     {[pinned]setenv}
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[build-system]
+build-backend = 'setuptools.build_meta'
\ No newline at end of file

From a93a63c208af1d13d5ea84623d160337c7fec6c5 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 17:05:49 -0300
Subject: [PATCH 56/64] fix: move import to inside function

---
 setup.py               | 3 ++-
 tests/requirements.txt | 1 -
 tox.ini                | 3 ---
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index f918db09e..dfe5b80ec 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,5 @@
 from pathlib import Path
 
-from packaging.version import parse as parse_version
 from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
@@ -15,6 +14,8 @@ def has_environment_marker_platform_impl_support():
     it is 18.5, see:
     https://setuptools.readthedocs.io/en/latest/history.html#id235
     """
+    from packaging.version import parse as parse_version
+
     return parse_version(setuptools_version) >= parse_version("18.5")
 
 
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 72350b216..618949795 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -14,4 +14,3 @@ brotli  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
-packaging
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 79d692599..ec3a59366 100644
--- a/tox.ini
+++ b/tox.ini
@@ -218,6 +218,3 @@ setenv =
     {[pinned]setenv}
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
-
-[build-system]
-build-backend = 'setuptools.build_meta'
\ No newline at end of file

From 0b1da44a05cc64970aa11ccc4d7a4a3bec143443 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 17:14:21 -0300
Subject: [PATCH 57/64] chore: Remove deprecated code

---
 setup.py | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/setup.py b/setup.py
index dfe5b80ec..1f214571b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,24 +1,10 @@
 from pathlib import Path
 
-from setuptools import __version__ as setuptools_version
 from setuptools import find_packages, setup
 
 version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
 
 
-def has_environment_marker_platform_impl_support():
-    """Code extracted from 'pytest/setup.py'
-    https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31
-
-    The first known release to support environment marker with range operators
-    it is 18.5, see:
-    https://setuptools.readthedocs.io/en/latest/history.html#id235
-    """
-    from packaging.version import parse as parse_version
-
-    return parse_version(setuptools_version) >= parse_version("18.5")
-
-
 install_requires = [
     "Twisted>=18.9.0",
     "cryptography>=36.0.0",
@@ -37,19 +23,10 @@ install_requires = [
     "tldextract",
     "lxml>=4.4.1",
 ]
-extras_require = {}
-cpython_dependencies = [
-    "PyDispatcher>=2.0.5",
-]
-if has_environment_marker_platform_impl_support():
-    extras_require[
-        ':platform_python_implementation == "CPython"'
-    ] = cpython_dependencies
-    extras_require[':platform_python_implementation == "PyPy"'] = [
-        "PyPyDispatcher>=2.1.0",
-    ]
-else:
-    install_requires.extend(cpython_dependencies)
+extras_require = {
+    ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],
+    ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],
+}
 
 
 setup(

From 82cf00bbc931723320c9aca3cf0305372027d0a0 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 20 Jun 2023 18:27:03 -0300
Subject: [PATCH 58/64] fix: default value

---
 scrapy/cmdline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index cfa771104..efc9b36ea 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -48,7 +48,7 @@ def _get_commands_from_module(module, inproject):
 
 def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     cmds = {}
-    for entry_point in entry_points(group):
+    for entry_point in entry_points().get(group, {}):
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()

From ee215a29704adbc61a40baeca1d27f6179a1735e Mon Sep 17 00:00:00 2001
From: Aaron Smith <60046611+medic-code@users.noreply.github.com>
Date: Wed, 21 Jun 2023 19:05:39 +0100
Subject: [PATCH 59/64] Change redirect text from Response.request docs (#5937)

---
 docs/topics/request-response.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 407df32d2..41df51589 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1103,9 +1103,10 @@ Response objects
         through all :ref:`Downloader Middlewares <topics-downloader-middleware>`.
         In particular, this means that:
 
-        - HTTP redirections will cause the original request (to the URL before
-          redirection) to be assigned to the redirected response (with the final
-          URL after redirection).
+        - HTTP redirections will create a new request from the request before
+          redirection. It has the majority of the same metadata and original
+          request attributes and gets assigned to the redirected response
+          instead of the propagation of the original request.
 
         - Response.request.url doesn't always equal Response.url
 

From 5360ba34bc345667f77a4d4256f15fd648e42e18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com>
Date: Wed, 21 Jun 2023 11:08:53 -0700
Subject: [PATCH 60/64] IOError and other cleanup (#4716)

---
 docs/utils/linkfix.py                         | 2 +-
 scrapy/downloadermiddlewares/decompression.py | 4 ++--
 scrapy/downloadermiddlewares/httpcache.py     | 2 +-
 scrapy/settings/default_settings.py           | 4 ++--
 scrapy/utils/gz.py                            | 2 +-
 scrapy/utils/python.py                        | 2 +-
 tests/test_downloader_handlers.py             | 2 +-
 tests/test_downloadermiddleware.py            | 4 ++--
 tests/test_mail.py                            | 2 --
 tests/test_robotstxt_interface.py             | 1 -
 tests/test_utils_gz.py                        | 2 +-
 tests/test_utils_iterators.py                 | 4 ++--
 12 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 1f270837c..c17b9d511 100644
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -30,7 +30,7 @@ def main():
     try:
         with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
             output_lines = out.readlines()
-    except IOError:
+    except OSError:
         print("linkcheck output not found; please run linkcheck first.")
         sys.exit(1)
 
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 5839dc243..3b8702419 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -63,7 +63,7 @@ class DecompressionMiddleware:
         archive = BytesIO(response.body)
         try:
             body = gzip.GzipFile(fileobj=archive).read()
-        except IOError:
+        except OSError:
             return
 
         respcls = responsetypes.from_args(body=body)
@@ -72,7 +72,7 @@ class DecompressionMiddleware:
     def _is_bzip2(self, response):
         try:
             body = bz2.decompress(response.body)
-        except IOError:
+        except OSError:
             return
 
         respcls = responsetypes.from_args(body=body)
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index b9316c43a..ac87d4a4e 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -37,7 +37,7 @@ class HttpCacheMiddleware:
         ConnectionLost,
         TCPTimedOutError,
         ResponseFailed,
-        IOError,
+        OSError,
     )
 
     def __init__(self, settings: Settings, stats: StatsCollector) -> None:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 89837b4ab..a4cb555bd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -268,9 +268,9 @@ RETRY_EXCEPTIONS = [
     "twisted.internet.error.ConnectionLost",
     "twisted.internet.error.TCPTimedOutError",
     "twisted.web.client.ResponseFailed",
-    # IOError is raised by the HttpCompression middleware when trying to
+    # OSError is raised by the HttpCompression middleware when trying to
     # decompress an empty response
-    IOError,
+    OSError,
     "scrapy.core.downloader.handlers.http11.TunnelError",
 ]
 
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index e5df34d2e..77e0197d8 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -15,7 +15,7 @@ def gunzip(data):
         try:
             chunk = f.read1(8196)
             output_list.append(chunk)
-        except (IOError, EOFError, struct.error):
+        except (OSError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
             # some pages are quite small so output_list is empty and f.extrabuf
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 27816c0df..bb5dbebbc 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -291,7 +291,7 @@ def without_none_values(iterable):
     try:
         return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
-        return type(iterable)((v for v in iterable if v is not None))
+        return type(iterable)(v for v in iterable if v is not None)
 
 
 def global_object_name(obj):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index fd4176e2f..9731b62c4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase):
     def test_non_existent(self):
         request = Request(f"file://{self.mktemp()}")
         d = self.download_request(request, Spider("foo"))
-        return self.assertFailure(d, IOError)
+        return self.assertFailure(d, OSError)
 
 
 class ContentLengthHeaderResource(resource.Resource):
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 060cfe08b..062e8a8b4 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase):
         In particular when some website returns a 30x response with header
         'Content-Encoding: gzip' giving as result the error below:
 
-            exceptions.IOError: Not a gzipped file
+            BadGzipFile: Not a gzipped file (...)
 
         """
         req = Request("http://example.com")
@@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase):
                 "Location": "http://example.com/login",
             },
         )
-        self.assertRaises(IOError, self._download, request=req, response=resp)
+        self.assertRaises(OSError, self._download, request=req, response=resp)
 
 
 class ResponseFromProcessRequestTest(ManagerTestCase):
diff --git a/tests/test_mail.py b/tests/test_mail.py
index bc7298e9d..504c78486 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -1,5 +1,3 @@
-# coding=utf-8
-
 import unittest
 from email.charset import Charset
 from io import BytesIO
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 8d87a322a..d7a923085 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,4 +1,3 @@
-# coding=utf-8
 from twisted.trial import unittest
 
 
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 6b2a458bc..7b7a25db8 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_no_gzip_file_raises(self):
         self.assertRaises(
-            IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
+            OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
         )
 
     def test_gunzip_truncated_short(self):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index faf7d2709..3598fa0bb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
-            self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
+            self.assertTrue(all(isinstance(k, str) for k in result_row.keys()))
+            self.assertTrue(all(isinstance(v, str) for v in result_row.values()))
 
     def test_csviter_delimiter(self):
         body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")

From 04ee3303e4487270a433f5c3a087bda9a87d7008 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Wed, 21 Jun 2023 22:04:06 -0700
Subject: [PATCH 61/64] Adding support for Windows of absolute pathlib.Path
 objects in FeedExporter (#5939)

---
 docs/topics/feed-exports.rst    |  4 ++--
 scrapy/extensions/feedexport.py |  7 +++++--
 tests/test_feedexport.py        | 25 +++++++++++++++++++++++++
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index b31dc069e..aba47d998 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -156,8 +156,8 @@ The feeds are stored in the local filesystem.
 -   Required external libraries: none
 
 Note that for the local filesystem storage (only) you can omit the scheme if
-you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
-systems though.
+you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
+Alternatively you can also use a :class:`pathlib.Path` object.
 
 .. _topics-feed-storage-ftp:
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 39934cbf3..1cdc78f59 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -382,7 +382,9 @@ class FeedExporter:
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-            uri = str(self.settings["FEED_URI"])  # handle pathlib.Path objects
+            uri = self.settings["FEED_URI"]
+            # handle pathlib.Path objects
+            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
             feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
@@ -392,7 +394,8 @@ class FeedExporter:
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
         for uri, feed_options in self.settings.getdict("FEEDS").items():
-            uri = str(uri)  # handle pathlib.Path objects
+            # handle pathlib.Path objects
+            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
             )
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 62a5697cd..8df86dbd7 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase):
         with self.assertRaises(NotConfigured):
             FeedExporter.from_crawler(crawler)
 
+    def test_absolute_pathlib_as_uri(self):
+        with tempfile.NamedTemporaryFile(suffix="json") as tmp:
+            settings = {
+                "FEEDS": {
+                    Path(tmp.name).resolve(): {
+                        "format": "json",
+                    },
+                },
+            }
+            crawler = get_crawler(settings_dict=settings)
+            exporter = FeedExporter.from_crawler(crawler)
+            self.assertIsInstance(exporter, FeedExporter)
+
+    def test_relative_pathlib_as_uri(self):
+        settings = {
+            "FEEDS": {
+                Path("./items.json"): {
+                    "format": "json",
+                },
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        exporter = FeedExporter.from_crawler(crawler)
+        self.assertIsInstance(exporter, FeedExporter)
+
 
 class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
     def __init__(self, uri):

From 080b9bd0b8f65dcf09ea8ad94505fec6c77f820c Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 22 Jun 2023 23:58:03 -0300
Subject: [PATCH 62/64] chore: Implement `pop` method on `BaseSettings` class

---
 scrapy/settings/__init__.py     | 14 ++++++++++++++
 tests/test_settings/__init__.py | 13 +++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index a3b849f7b..57fe1d17a 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -75,6 +75,8 @@ class BaseSettings(MutableMapping):
     highest priority will be retrieved.
     """
 
+    __default = object()
+
     def __init__(self, values=None, priority="project"):
         self.frozen = False
         self.attributes = {}
@@ -445,6 +447,18 @@ class BaseSettings(MutableMapping):
         else:
             p.text(pformat(self.copy_to_dict()))
 
+    def pop(self, name, default=__default):
+        try:
+            value = self.attributes[name]
+        except KeyError:
+            if default is self.__default:
+                raise
+
+            return SettingsAttribute(default, get_settings_priority("project"))
+        else:
+            del self.attributes[name]
+            return value
+
 
 class Settings(BaseSettings):
     """
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 4a577cd8c..0e2f4aa98 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -451,6 +451,19 @@ class SettingsTest(unittest.TestCase):
         self.assertIsInstance(myhandler_instance, FileDownloadHandler)
         self.assertTrue(hasattr(myhandler_instance, "download_request"))
 
+    def test_pop_item_with_default_value(self):
+        settings = Settings()
+
+        with self.assertRaises(KeyError):
+            settings.pop("DUMMY_CONFIG")
+
+        dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value")
+
+        self.assertEqual(
+            repr(dummy_config), "<SettingsAttribute value='dummy_value' priority=20>"
+        )
+        self.assertEqual(dummy_config.value, "dummy_value")
+
 
 if __name__ == "__main__":
     unittest.main()

From 876feaf339e181c9c5a6b9a5f8ffedc03a9ed3d2 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 23 Jun 2023 00:14:31 -0300
Subject: [PATCH 63/64] chore: Use dunder to delete item instead of del keyword
 to handle immutable settings

---
 scrapy/settings/__init__.py     |  2 +-
 tests/test_settings/__init__.py | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 57fe1d17a..8b3bdbabe 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -456,7 +456,7 @@ class BaseSettings(MutableMapping):
 
             return SettingsAttribute(default, get_settings_priority("project"))
         else:
-            del self.attributes[name]
+            self.__delitem__(name)
             return value
 
 
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 0e2f4aa98..125b1d96f 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -464,6 +464,22 @@ class SettingsTest(unittest.TestCase):
         )
         self.assertEqual(dummy_config.value, "dummy_value")
 
+    def test_pop_item_with_frozen_settings(self):
+        settings = Settings(
+            {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
+        )
+
+        self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value")
+
+        settings.freeze()
+
+        with self.assertRaises(TypeError) as error:
+            settings.pop("OTHER_DUMMY_CONFIG")
+
+        self.assertEqual(
+            str(error.exception), "Trying to modify an immutable Settings object"
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

From a3f8912d69eacdd2208617e6afb418e4e1847e36 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 23 Jun 2023 00:15:32 -0300
Subject: [PATCH 64/64] chore: Rename test

---
 tests/test_settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 125b1d96f..bb6dc67fa 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -464,7 +464,7 @@ class SettingsTest(unittest.TestCase):
         )
         self.assertEqual(dummy_config.value, "dummy_value")
 
-    def test_pop_item_with_frozen_settings(self):
+    def test_pop_item_with_immutable_settings(self):
         settings = Settings(
             {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
         )