From 8d97f49e5e31955fb972dc63106298032ebe0e59 Mon Sep 17 00:00:00 2001
From: Harrison Gregg <animasterh@gmail.com>
Date: Sat, 9 Sep 2017 13:54:48 +0630
Subject: [PATCH 01/58] Re-add SIGINT handler in inspect_response after shell
 closes

---
 scrapy/shell.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scrapy/shell.py b/scrapy/shell.py
index 80b625633..af91d6ce0 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -164,7 +164,11 @@ class Shell(object):
 
 def inspect_response(response, spider):
     """Open a shell to inspect the given response"""
+    # Shell.start removes the SIGINT handler, so save it and re-add it after
+    # the shell has closed
+    sigint_handler = signal.getsignal(signal.SIGINT)
     Shell(spider.crawler).start(response=response, spider=spider)
+    signal.signal(signal.SIGINT, sigint_handler)
 
 
 def _request_deferred(request):

From 0beed7055cdec3683bf67ec01b573ddf87df723f Mon Sep 17 00:00:00 2001
From: Silvio Pavanetto <silvio.pavanetto@gmail.com>
Date: Wed, 23 Mar 2022 17:28:55 +0100
Subject: [PATCH 02/58] fix: return unique_list only when link_extractor.unique
 is True

---
 scrapy/linkextractors/lxmlhtml.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index b5d2585a8..caef504a0 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -161,4 +161,6 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         for doc in docs:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
-        return unique_list(all_links)
+        if self.link_extractor.unique:
+            return unique_list(all_links)
+        return all_links

From 34e4ed72ea87601ce17c8d248d6a54d3e8b73194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 21 Jun 2022 12:46:54 +0200
Subject: [PATCH 03/58] Document how DOWNLOAD_DELAY affects per-domain
 concurrency

---
 docs/topics/settings.rst | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2046c6446..468975d6a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -633,25 +633,41 @@ DOWNLOAD_DELAY
 
 Default: ``0``
 
-The amount of time (in secs) that the downloader should wait before downloading
-consecutive pages from the same website. This can be used to throttle the
-crawling speed to avoid hitting servers too hard. Decimal numbers are
-supported.  Example::
+Minimum seconds to wait between 2 consecutive requests to the same domain.
 
-    DOWNLOAD_DELAY = 0.25    # 250 ms of delay
+Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
+servers too hard.
+
+Decimal numbers are supported. For example, to send a maximum of 4 requests
+every 10 seconds::
+
+    DOWNLOAD_DELAY = 2.5
 
 This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
-setting (which is enabled by default). By default, Scrapy doesn't wait a fixed
-amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`.
+setting, which is enabled by default.
 
 When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
-per ip address instead of per domain.
+per IP address instead of per domain.
+
+Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
+concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
+time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
+concurrency for that domain is 1. When testing throttling configurations, it
+usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first,
+and only increase :setting:`DOWNLOAD_DELAY` once
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is
+desired.
 
 .. _spider-download_delay-attribute:
 
-You can also change this setting per spider by setting ``download_delay``
+You can change this setting per spider by setting the ``download_delay``
 spider attribute.
 
+It is also possible to change this setting per domain, although it requires
+non-trivial code. See the implementation of the :ref:`AutoThrottle
+<topics-autothrottle>` extension for an example.
+
+
 .. setting:: DOWNLOAD_HANDLERS
 
 DOWNLOAD_HANDLERS

From 087334009c2adcf45cb224b28c9c40857253657a Mon Sep 17 00:00:00 2001
From: Matt Mayfield <mattmayf@umich.edu>
Date: Sun, 11 Dec 2022 23:12:41 -0500
Subject: [PATCH 04/58] Call `finish_exporting` even when itemcount == 0

---
 scrapy/extensions/feedexport.py |  2 +-
 tests/test_feedexport.py        | 57 ++++++++++++++++++++++++++++++++-
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0aa27e417..c3382d9d8 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -350,11 +350,11 @@ class FeedExporter:
         return defer.DeferredList(deferred_list) if deferred_list else None
 
     def _close_slot(self, slot, spider):
+        slot.finish_exporting()
         if not slot.itemcount and not slot.store_empty:
             # We need to call slot.storage.store nonetheless to get the file
             # properly closed.
             return defer.maybeDeferred(slot.storage.store, slot.file)
-        slot.finish_exporting()
         logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
         d = defer.maybeDeferred(slot.storage.store, slot.file)
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 97c3a74b3..d33ec281f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -33,8 +33,9 @@ from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.exporters import CsvItemExporter
+from scrapy.exporters import CsvItemExporter, JsonItemExporter
 from scrapy.extensions.feedexport import (
+    _FeedSlot,
     BlockingFeedStorage,
     FeedExporter,
     FileFeedStorage,
@@ -890,6 +891,60 @@ class FeedExportTest(FeedExportTestBase):
             data = yield self.exported_no_data(settings)
             self.assertEqual(b'', data[fmt])
 
+    @defer.inlineCallbacks
+    def test_finish_exporting_is_called(self):
+        # for each format, keep track of when start_exporting
+        # has been called but finish_exporting hasn't been called
+        startRecordingTracker = {}
+        # we expect finish_recording to be called, setting this to false
+        expected = {'json': False}
+
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+        ]
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+            },
+            'FEED_EXPORT_INDENT': None,
+        }
+
+        # override export_item to raise exception
+        class FakeJsonItemExporter(JsonItemExporter):
+            def export_item(self, item):
+                raise Exception('foo')
+
+        # override start/stop_exporting to modify startRecordingTracker
+        class FakeFeedSlot(_FeedSlot):
+            def start_exporting(self):
+                startRecordingTracker[self.format] = True
+                if not self._exporting:
+                    self.exporter.start_exporting()
+                    self._exporting = True
+            
+            def finish_exporting(self):
+                print('finish export called')
+                startRecordingTracker[self.format] = False
+                if self._exporting:
+                    self.exporter.finish_exporting()
+                    self._exporting = False
+            
+
+        with ExitStack() as stack:
+            stack.enter_context(
+                mock.patch(
+                    'scrapy.exporters.JsonItemExporter', FakeJsonItemExporter
+                )
+            )
+            stack.enter_context(
+                mock.patch(
+                    'scrapy.extensions.feedexport._FeedSlot', FakeFeedSlot
+                )
+            )
+            _ = yield self.exported_data(items, settings)
+            self.assertDictEqual(startRecordingTracker, expected)
+
+
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):
         formats = (

From 66f127eb37ac7d2d85d641f9d8f49fa6e3130a92 Mon Sep 17 00:00:00 2001
From: Matt Mayfield <mattmayf@umich.edu>
Date: Mon, 12 Dec 2022 11:46:05 -0500
Subject: [PATCH 05/58] Make test cleaner and more reusable

---
 tests/test_feedexport.py | 84 ++++++++++++++++++++++------------------
 1 file changed, 46 insertions(+), 38 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d33ec281f..a6356aecc 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -680,6 +680,45 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                     break
         return result
 
+class InstrumentedFeedSlot(_FeedSlot):
+    """Instrumented _FeedSlot subclass for keeping track of calls to
+    start_exporting and finish_exporting."""
+    def start_exporting(self):
+        self.update_listener('start')
+        super().start_exporting()
+    
+    def finish_exporting(self):
+        self.update_listener('finish')
+        super().start_exporting()
+    
+    @classmethod
+    def subscribe__listener(cls, listener):
+        cls.update_listener = listener.update
+
+class IsExportingListener:
+    """When subscribed to InstrumentedFeedSlot, keeps track of when
+    a call to start_exporting has been made without a closing call to
+    finish_exporting and when a call to finis_exporting has been made
+    before a call to start_exporting."""
+    def __init__(self):
+        self.start_without_finish = False
+        self.finish_without_start = False
+
+    def update(self, method):
+        if method == 'start':
+            self.start_without_finish = True
+        elif method == 'finish':
+            if self.start_without_finish:
+                self.start_without_finish = False
+            else:
+                self.finish_before_start = True
+
+
+class ExceptionJsonItemExporter(JsonItemExporter):
+    """JsonItemExporter that throws an exception every time export_item is called."""
+    def export_item(self, _):
+        raise Exception('foo')
+
 
 class FeedExportTest(FeedExportTestBase):
     __test__ = True
@@ -893,12 +932,6 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_finish_exporting_is_called(self):
-        # for each format, keep track of when start_exporting
-        # has been called but finish_exporting hasn't been called
-        startRecordingTracker = {}
-        # we expect finish_recording to be called, setting this to false
-        expected = {'json': False}
-
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
         ]
@@ -906,43 +939,18 @@ class FeedExportTest(FeedExportTestBase):
             'FEEDS': {
                 self._random_temp_filename(): {'format': 'json'},
             },
+            'FEED_EXPORTERS': {'json': ExceptionJsonItemExporter},
             'FEED_EXPORT_INDENT': None,
         }
 
-        # override export_item to raise exception
-        class FakeJsonItemExporter(JsonItemExporter):
-            def export_item(self, item):
-                raise Exception('foo')
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
 
-        # override start/stop_exporting to modify startRecordingTracker
-        class FakeFeedSlot(_FeedSlot):
-            def start_exporting(self):
-                startRecordingTracker[self.format] = True
-                if not self._exporting:
-                    self.exporter.start_exporting()
-                    self._exporting = True
-            
-            def finish_exporting(self):
-                print('finish export called')
-                startRecordingTracker[self.format] = False
-                if self._exporting:
-                    self.exporter.finish_exporting()
-                    self._exporting = False
-            
-
-        with ExitStack() as stack:
-            stack.enter_context(
-                mock.patch(
-                    'scrapy.exporters.JsonItemExporter', FakeJsonItemExporter
-                )
-            )
-            stack.enter_context(
-                mock.patch(
-                    'scrapy.extensions.feedexport._FeedSlot', FakeFeedSlot
-                )
-            )
+        with mock.patch('scrapy.extensions.feedexport._FeedSlot',
+                        InstrumentedFeedSlot):
             _ = yield self.exported_data(items, settings)
-            self.assertDictEqual(startRecordingTracker, expected)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
 
 
     @defer.inlineCallbacks

From 8d67a08155cfd0b745a2b62538d7fd15c033184e Mon Sep 17 00:00:00 2001
From: Matt Mayfield <mattmayf@umich.edu>
Date: Mon, 12 Dec 2022 11:55:42 -0500
Subject: [PATCH 06/58] Change test name and add additional tests

---
 tests/test_feedexport.py | 61 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 60 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index a6356aecc..4d533142b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -931,7 +931,47 @@ class FeedExportTest(FeedExportTestBase):
             self.assertEqual(b'', data[fmt])
 
     @defer.inlineCallbacks
-    def test_finish_exporting_is_called(self):
+    def test_start_finish_exporting_items(self):
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+        ]
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+            },
+            'FEED_EXPORT_INDENT': None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch('scrapy.extensions.feedexport._FeedSlot',
+                        InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
+
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_no_items(self):
+        items = []
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+            },
+            'FEED_EXPORT_INDENT': None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch('scrapy.extensions.feedexport._FeedSlot',
+                        InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
+
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_items_exception(self):
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
         ]
@@ -952,6 +992,25 @@ class FeedExportTest(FeedExportTestBase):
             self.assertFalse(listener.start_without_finish)
             self.assertFalse(listener.finish_without_start)
 
+    @defer.inlineCallbacks
+    def test_start_finish_exporting_no_items_exception(self):
+        items = []
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {'format': 'json'},
+            },
+            'FEED_EXPORTERS': {'json': ExceptionJsonItemExporter},
+            'FEED_EXPORT_INDENT': None,
+        }
+
+        listener = IsExportingListener()
+        InstrumentedFeedSlot.subscribe__listener(listener)
+
+        with mock.patch('scrapy.extensions.feedexport._FeedSlot',
+                        InstrumentedFeedSlot):
+            _ = yield self.exported_data(items, settings)
+            self.assertFalse(listener.start_without_finish)
+            self.assertFalse(listener.finish_without_start)
 
     @defer.inlineCallbacks
     def test_export_no_items_store_empty(self):

From 40f4b262d2046f8b64d55c477f1a8c9897d2c838 Mon Sep 17 00:00:00 2001
From: Matt Mayfield <mattmayf@umich.edu>
Date: Mon, 12 Dec 2022 12:36:29 -0500
Subject: [PATCH 07/58] Fix style errors

---
 tests/test_feedexport.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 4d533142b..0d5b0f08e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -680,21 +680,23 @@ class FeedExportTestBase(ABC, unittest.TestCase):
                     break
         return result
 
+
 class InstrumentedFeedSlot(_FeedSlot):
     """Instrumented _FeedSlot subclass for keeping track of calls to
     start_exporting and finish_exporting."""
     def start_exporting(self):
         self.update_listener('start')
         super().start_exporting()
-    
+
     def finish_exporting(self):
         self.update_listener('finish')
         super().start_exporting()
-    
+
     @classmethod
     def subscribe__listener(cls, listener):
         cls.update_listener = listener.update
 
+
 class IsExportingListener:
     """When subscribed to InstrumentedFeedSlot, keeps track of when
     a call to start_exporting has been made without a closing call to

From f3c6bfdebe2f8ee3cfaec172ab8607f75829fd43 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:10 -0500
Subject: [PATCH 08/58] spelling: accounting

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index c97de0ed8..dd51b11da 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5082,7 +5082,7 @@ Scrapy changes:
 - promoted :ref:`topics-djangoitem` to main contrib
 - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
 - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
-- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
+- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
 - removed signal: ``scrapy.mail.mail_sent``
 - removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
 - DBM is now the default storage backend for HTTP cache middleware

From 334f844e58edfbd791fee6697e3259dbb5a741a5 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:09:18 -0500
Subject: [PATCH 09/58] spelling: and

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 scrapy/cmdline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 8218a51c8..88936c276 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -24,7 +24,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
 
 
 def _iter_command_classes(module_name):
-    # TODO: add `name` attribute to commands and and merge this function with
+    # TODO: add `name` attribute to commands and merge this function with
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
         for obj in vars(module).values():

From 226c42ad1423d92ae9b6aa6e05f4e5d5cdd9f3f8 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:11 -0500
Subject: [PATCH 10/58] spelling: canonicalize

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 sep/sep-016.rst | 2 +-
 sep/sep-018.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sep/sep-016.rst b/sep/sep-016.rst
index 335f09f45..a60ab30dd 100644
--- a/sep/sep-016.rst
+++ b/sep/sep-016.rst
@@ -148,7 +148,7 @@ Another example could be for building URL canonicalizers:
 ::
 
    #!python
-   class CanonializeUrl(LegSpider):
+   class CanonicalizeUrl(LegSpider):
 
        def process_request(self, request):
            curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index fe707923a..fa4f7da82 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -321,7 +321,7 @@ Another example could be for building URL canonicalizers:
 ::
 
    #!python
-   class CanonializeUrl(object):
+   class CanonicalizeUrl(object):
 
        def process_request(self, request, response, spider):
            curl = canonicalize_url(request.url, 

From 1300c1c8816c86e8ce9cff114dcefdf785138018 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 21:33:14 -0500
Subject: [PATCH 11/58] spelling: children

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_utils_deprecate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 50c63dfab..9fbb07472 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -69,7 +69,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
         self.assertIn('foo.NewClass', str(w[1].message))
         self.assertIn('bar.OldClass', str(w[1].message))
 
-    def test_subclassing_warns_only_on_direct_childs(self):
+    def test_subclassing_warns_only_on_direct_children(self):
         Deprecated = create_deprecated_class('Deprecated', NewName,
                                              warn_once=False,
                                              warn_category=MyWarning)

From 87fc92441f39df4eb1a43482f1475e2ba5415130 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:11 -0500
Subject: [PATCH 12/58] spelling: crawlable

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 docs/news.rst       | 8 ++++----
 scrapy/utils/url.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index dd51b11da..4609c5158 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -4725,7 +4725,7 @@ Enhancements
 - [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
   To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
 - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
-- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
+- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`)
 - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
 - Selectors register EXSLT namespaces by default (:issue:`472`)
 - Unify item loaders similar to selectors renaming (:issue:`461`)
@@ -4905,7 +4905,7 @@ Scrapy 0.18.0 (released 2013-08-09)
 -----------------------------------
 
 - Lot of improvements to testsuite run using Tox, including a way to test on pypi
-- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
+- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`)
 - Use lxml recover option to parse sitemaps (:issue:`347`)
 - Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
 - Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
@@ -5148,7 +5148,7 @@ Scrapy 0.14
 New features and settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Support for `AJAX crawleable urls`_
+- Support for `AJAX crawlable urls`_
 - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
 - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
 - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
@@ -5419,7 +5419,7 @@ Scrapy 0.7
 First release of Scrapy.
 
 
-.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
+.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
 .. _botocore: https://github.com/boto/botocore
 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 21201ace5..6c59af6db 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -46,7 +46,7 @@ def parse_url(url, encoding=None):
 
 def escape_ajax(url):
     """
-    Return the crawleable url according to:
+    Return the crawlable url according to:
     https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
 
     >>> escape_ajax("www.example.com/ajax.html#!key=value")

From 581eb2d1b4fe89953115859cc6cff1cbc30bc700 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:12 -0500
Subject: [PATCH 13/58] spelling: downloader

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 sep/sep-021.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sep/sep-021.rst b/sep/sep-021.rst
index c1ec16f7f..d56bc26af 100644
--- a/sep/sep-021.rst
+++ b/sep/sep-021.rst
@@ -79,7 +79,7 @@ If it raises an exception, Scrapy will print it and exit.
 Examples::
 
     def addon_configure(settings):
-        settings.overrides['DOWNLADER_MIDDLEWARES'].update({
+        settings.overrides['DOWNLOADER_MIDDLEWARES'].update({
             'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
         })
 

From e6ebadcd54e018dc1f90505bcdde024dba813a80 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:12 -0500
Subject: [PATCH 14/58] spelling: freshness

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 scrapy/extensions/httpcache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 4e76fe5e5..b8d8e94dc 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -190,7 +190,7 @@ class RFC2616Policy:
         if response.status in (300, 301, 308):
             return self.MAXAGE
 
-        # Insufficient information to compute fresshness lifetime
+        # Insufficient information to compute freshness lifetime
         return 0
 
     def _compute_current_age(self, response, request, now):

From b6426b8e03759f3364da88f52406aa8cf8dbf92c Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:13 -0500
Subject: [PATCH 15/58] spelling: heavily

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 4609c5158..ee7462d5c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5408,7 +5408,7 @@ Backward-incompatible changes
 - Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
 - Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
 - Refactored HTTP Cache middleware
-- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
+- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
 - Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
 - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
 - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)

From a26b6b0607026e4ea266ed3e30d2643d5990bf8a Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:13 -0500
Subject: [PATCH 16/58] spelling: indistinct

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_http_response.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 74e170ec0..bb16b8904 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -425,7 +425,7 @@ class TextResponseTest(BaseResponseTest):
     def test_bom_is_removed_from_body(self):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
-        # response.text in indistint order doesn't affect final
+        # response.text in indistinct order doesn't affect final
         # values for encoding and decoded body.
         url = 'http://example.com'
         body = b"\xef\xbb\xbfWORD"

From b62aacfee36397e8173cf3370a2088f3bb53943f Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:14 -0500
Subject: [PATCH 17/58] spelling: initializing

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 docs/topics/spiders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index ffe41cf3e..7aa8555d5 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -99,7 +99,7 @@ scrapy.Spider
    .. attribute:: crawler
 
       This attribute is set by the :meth:`from_crawler` class method after
-      initializating the class, and links to the
+      initializing the class, and links to the
       :class:`~scrapy.crawler.Crawler` object to which this spider instance is
       bound.
 

From e894db2f3f7d0873c38542744fde946a6db0d659 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:14 -0500
Subject: [PATCH 18/58] spelling: laziness

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_crawl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 5ec96e4a7..4f7c24e27 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -177,7 +177,7 @@ class CrawlTestCase(TestCase):
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
-    def test_start_requests_lazyness(self):
+    def test_start_requests_laziness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = get_crawler(BrokenStartRequestsSpider, settings)
         yield crawler.crawl(mockserver=self.mockserver)

From fff2f2db20c7a147fb2631bd6deca03a74e7e208 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:15 -0500
Subject: [PATCH 19/58] spelling: measure

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 extras/qpsclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 28703650d..2b501d11d 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -1,5 +1,5 @@
 """
-A spider that generate light requests to meassure QPS throughput
+A spider that generate light requests to measure QPS throughput
 
 usage:
 

From 5f33a64a02843ca595e477756cdf970e2f944278 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:15 -0500
Subject: [PATCH 20/58] spelling: middleware

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 docs/news.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index ee7462d5c..a1e3e25e0 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -4939,8 +4939,8 @@ Scrapy 0.18.0 (released 2013-08-09)
 - Added ``--pdb`` option to ``scrapy`` command line tool
 - Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
 - Several improvements to spider contracts
-- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
-- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
+- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections,
+- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62
 - added from_crawler method to spiders
 - added system tests with mock server
 - more improvements to macOS compatibility (thanks Alex Cepoi)

From a839b61147da92ad7b175919fb31fae8293cec97 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:09:28 -0500
Subject: [PATCH 21/58] spelling: nonexistent

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_crawl.py               | 2 +-
 tests/test_downloader_handlers.py | 4 ++--
 tests/test_http_request.py        | 6 +++---
 tests/test_utils_python.py        | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 4f7c24e27..44ced42f5 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -150,7 +150,7 @@ class CrawlTestCase(TestCase):
             raise unittest.SkipTest("Non-existing hosts are resolvable")
         crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
-            # try to fetch the homepage of a non-existent domain
+            # try to fetch the homepage of a nonexistent domain
             yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
         self._assert_retried(log)
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 3dc2745a0..433b6d0c8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1050,8 +1050,8 @@ class BaseFTPTestCase(unittest.TestCase):
 
         return self._add_test_callbacks(d, _test)
 
-    def test_ftp_download_notexist(self):
-        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt",
+    def test_ftp_download_nonexistent(self):
+        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/nonexistent.txt",
                           meta=self.req_meta)
         d = self.download_handler.download_request(request, None)
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9f7f1854f..c3b729b76 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -873,7 +873,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
 
-    def test_from_response_formname_notexist(self):
+    def test_from_response_formname_nonexistent(self):
         response = _buildresponse(
             """<form name="form1" action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
@@ -912,7 +912,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
 
-    def test_from_response_formname_notexists_fallback_formid(self):
+    def test_from_response_formname_nonexistent_fallback_formid(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
@@ -927,7 +927,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(r1)
         self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
 
-    def test_from_response_formid_notexist(self):
+    def test_from_response_formid_nonexistent(self):
         response = _buildresponse(
             """<form id="form1" action="post.php" method="POST">
             <input type="hidden" name="one" value="1">
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 403e4f8fe..8cca17aa5 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -160,7 +160,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         b = Obj()
         # no attributes given return False
         self.assertFalse(equal_attributes(a, b, []))
-        # not existent attributes
+        # nonexistent attributes
         self.assertFalse(equal_attributes(a, b, ['x', 'y']))
 
         a.x = 1

From 1a5cf00db78dc5173e75b85951f67c323b81ad96 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:17 -0500
Subject: [PATCH 22/58] spelling: overridden

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 scrapy/core/downloader/webclient.py     |  2 +-
 scrapy/pipelines/images.py              |  4 ++--
 tests/test_pipeline_images.py           |  4 ++--
 tests/test_request_attribute_binding.py | 16 ++++++++--------
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 255ca62e6..a261bf0ad 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -98,7 +98,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 # This class used to inherit from Twisted’s
 # twisted.web.client.HTTPClientFactory. When that class was deprecated in
 # Twisted (https://github.com/twisted/twisted/pull/643), we merged its
-# non-overriden code into this class.
+# non-overridden code into this class.
 class ScrapyHTTPClientFactory(ClientFactory):
 
     protocol = ScrapyHTTPPageGetter
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 6a28a3b87..05b8693ad 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -147,8 +147,8 @@ class ImagesPipeline(FilesPipeline):
         if self._deprecated_convert_image is None:
             self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image)
             if self._deprecated_convert_image:
-                warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, '
-                              'overriden method does not accept response_body argument.',
+                warnings.warn(f'{self.__class__.__name__}.convert_image() method overridden in a deprecated way, '
+                              'overridden method does not accept response_body argument.',
                               category=ScrapyDeprecationWarning)
 
         if self._deprecated_convert_image:
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f98d40fda..30166a502 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -173,8 +173,8 @@ class ImagesPipelineTestCase(unittest.TestCase):
                 self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
                 self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
 
-                expected_warning_msg = ('.convert_image() method overriden in a deprecated way, '
-                                        'overriden method does not accept response_body argument.')
+                expected_warning_msg = ('.convert_image() method overridden in a deprecated way, '
+                                        'overridden method does not accept response_body argument.')
                 self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1)
 
     def test_convert_image_old(self):
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 15e400327..c1ee4baee 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -11,12 +11,12 @@ from tests.mockserver import MockServer
 from tests.spiders import SingleRequestSpider
 
 
-OVERRIDEN_URL = "https://example.org"
+OVERRIDDEN_URL = "https://example.org"
 
 
 class ProcessResponseMiddleware:
     def process_response(self, request, response, spider):
-        return response.replace(request=Request(OVERRIDEN_URL))
+        return response.replace(request=Request(OVERRIDDEN_URL))
 
 
 class RaiseExceptionRequestMiddleware:
@@ -30,7 +30,7 @@ class CatchExceptionOverrideRequestMiddleware:
         return Response(
             url="http://localhost/",
             body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
-            request=Request(OVERRIDEN_URL),
+            request=Request(OVERRIDDEN_URL),
         )
 
 
@@ -52,7 +52,7 @@ class AlternativeCallbacksSpider(SingleRequestSpider):
 class AlternativeCallbacksMiddleware:
     def process_response(self, request, response, spider):
         new_request = request.replace(
-            url=OVERRIDEN_URL,
+            url=OVERRIDDEN_URL,
             callback=spider.alt_callback,
             cb_kwargs={"foo": "bar"},
         )
@@ -127,13 +127,13 @@ class CrawlTestCase(TestCase):
             yield crawler.crawl(seed=url, mockserver=self.mockserver)
 
         response = crawler.spider.meta["responses"][0]
-        self.assertEqual(response.request.url, OVERRIDEN_URL)
+        self.assertEqual(response.request.url, OVERRIDDEN_URL)
 
         self.assertEqual(signal_params["response"].url, url)
-        self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
+        self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL)
 
         log.check_present(
-            ("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)"),
+            ("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)"),
         )
 
     @defer.inlineCallbacks
@@ -154,7 +154,7 @@ class CrawlTestCase(TestCase):
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
-        self.assertEqual(response.request.url, OVERRIDEN_URL)
+        self.assertEqual(response.request.url, OVERRIDDEN_URL)
 
     @defer.inlineCallbacks
     def test_downloader_middleware_do_not_override_in_process_exception(self):

From f5d024f16c8c3aea6a7dc4e648e907dfa3bd496f Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:17 -0500
Subject: [PATCH 23/58] spelling: parsley

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 sep/sep-018.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index fa4f7da82..96df82f6b 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -594,18 +594,18 @@ A middleware to Scrape data using Parsley as described in UsingParsley
 
    class ParsleyExtractor(object):
 
-       def __init__(self, parslet_json_code):
-           parslet = json.loads(parselet_json_code)
+       def __init__(self, parsley_json_code):
+           parsley = json.loads(parselet_json_code)
            class ParsleyItem(Item):
                def __init__(self, *a, **kw):
-                   for name in parslet.keys():
+                   for name in parsley.keys():
                        self.fields[name] = Field()
                super(ParsleyItem, self).__init__(*a, **kw)
            self.item_class = ParsleyItem    
-           self.parsley = PyParsley(parslet, output='python') 
+           self.parsley = PyParsley(parsley, output='python') 
 
        def process_response(self, response, request, spider):
-           return self.item_class(self.parsly.parse(string=response.body))
+           return self.item_class(self.parsley.parse(string=response.body))
 
 
 

From 8ae77fdb34d862b4ad466f7f9764132502787c47 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:18 -0500
Subject: [PATCH 24/58] spelling: pipeline

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/pipelines.py   | 2 +-
 tests/test_engine.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/pipelines.py b/tests/pipelines.py
index fed2af7d3..c130f057f 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline:
         return item
 
 
-class ProcessWithZeroDivisionErrorPipiline:
+class ProcessWithZeroDivisionErrorPipeline:
 
     def process_item(self, item, spider):
         1 / 0
diff --git a/tests/test_engine.py b/tests/test_engine.py
index aa3313659..870993edf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -109,7 +109,7 @@ class DataClassItemsSpider(TestSpider):
 class ItemZeroDivisionErrorSpider(TestSpider):
     custom_settings = {
         "ITEM_PIPELINES": {
-            "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300,
+            "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
         }
     }
 

From 66ab82f12631e82c2c0426d68e52ac5e59344402 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:18 -0500
Subject: [PATCH 25/58] spelling: precedence

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_spidermiddleware_referer.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 63daf0b8a..6a2af0e7d 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -434,17 +434,17 @@ class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
+class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     req_meta = {'referrer_policy': POLICY_NO_REFERRER}
 
 
-class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
+class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
@@ -561,22 +561,22 @@ class TestSettingsPolicyByName(TestCase):
             RefererMiddleware(settings)
 
 
-class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
 
 
-class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
 
 
-class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
 
-class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
     """
     The empty string means "no-referrer-when-downgrade"
     """

From 826e0ee6111739d154176bfdabb32cbf13f1dd0c Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:09:34 -0500
Subject: [PATCH 26/58] spelling: preexisting

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 91476abf8..b8b481f20 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -316,7 +316,7 @@ class StartprojectTemplatesTest(ProjectTest):
         self.assertEqual(actual_permissions, expected_permissions)
 
     def test_startproject_permissions_unchanged_in_destination(self):
-        """Check that pre-existing folders and files in the destination folder
+        """Check that preexisting folders and files in the destination folder
         do not see their permissions modified."""
         scrapy_path = scrapy.__path__[0]
         project_template = Path(scrapy_path, 'templates', 'project')

From 6aa5374bd3924dc98047a704569f5e1fc87559f1 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:19 -0500
Subject: [PATCH 27/58] spelling: received

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_pipeline_images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 30166a502..f4d36aae9 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -209,7 +209,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
             self.assertEqual(converted.mode, 'RGB')
             self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
 
-            # ensure that we recieved deprecation warnings
+            # ensure that we received deprecation warnings
             expected_warning_msg = '.convert_image() method called in a deprecated way'
             self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4)
 

From d27c611cc0d2fd9786752c8996960dd23fd8e2ac Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:19 -0500
Subject: [PATCH 28/58] spelling: referrer

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_spidermiddleware_referer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 6a2af0e7d..df44a1c42 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -147,7 +147,7 @@ class MixinSameOrigin:
         ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
         ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
 
-        # Different protocols: do NOT send refferer
+        # Different protocols: do NOT send referrer
         ('https://example.com/page.html', 'http://example.com/not-page.html', None),
         ('https://example.com/page.html', 'http://not.example.com/', None),
         ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),

From 010cf9d42089f33dbc6408f034adab289becbea8 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:20 -0500
Subject: [PATCH 29/58] spelling: refresh

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 scrapy/utils/response.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 23bd2da65..7a6f4a96f 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -36,7 +36,7 @@ def get_meta_refresh(
     response: "scrapy.http.response.text.TextResponse",
     ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
-    """Parse the http-equiv refrsh parameter from the given response"""
+    """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
         _metaref_cache[response] = html.get_meta_refresh(

From 1e44d4614ec7d34c4fb8f50a3429e4b1ddeb2253 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:20 -0500
Subject: [PATCH 30/58] spelling: straight

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_pipeline_images.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f4d36aae9..6f5466191 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -182,7 +182,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter('always')
             SIZE = (100, 100)
-            # straigh forward case: RGB and JPEG
+            # straight forward case: RGB and JPEG
             COLOUR = (0, 127, 255)
             im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR)
             converted, _ = self.pipeline.convert_image(im)
@@ -216,7 +216,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
     def test_convert_image_new(self):
         # tests for new API
         SIZE = (100, 100)
-        # straigh forward case: RGB and JPEG
+        # straight forward case: RGB and JPEG
         COLOUR = (0, 127, 255)
         im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR)
         converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)

From 860fbef608445230c16d0ddd4c2b676866449862 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:21 -0500
Subject: [PATCH 31/58] spelling: unknown

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index c3b729b76..81cebdc7b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -355,7 +355,7 @@ class RequestTest(unittest.TestCase):
             )
             self.assertEqual(r.method, "DELETE")
 
-        # If `ignore_unknon_options` is set to `False` it raises an error with
+        # If `ignore_unknown_options` is set to `False` it raises an error with
         # the unknown options: --foo and -z
         self.assertRaises(
             ValueError,

From 2cb1e6668a51294df909b0ac5f7e35b7e858446e Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 20:26:21 -0500
Subject: [PATCH 32/58] spelling: workaround

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 8e0a7fae2..fd61a8e4e 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -191,7 +191,7 @@ def _select_value(ele: SelectElement, n: str, v: str):
         o = ele.value_options
         return (n, o[0]) if o else (None, None)
     if v is not None and multiple:
-        # This is a workround to bug in lxml fixed 2.3.1
+        # This is a workaround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
         selected_options = ele.xpath('.//option[@selected]')
         values = [(o.get('value') or o.text or '').strip() for o in selected_options]

From 8a526d161c0a7d4818f16f6a9e1ae563e16abab6 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 21:46:11 -0500
Subject: [PATCH 33/58] spelling: user

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 tests/test_downloadermiddleware_useragent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index a286764fd..c97e28cb8 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -20,7 +20,7 @@ class UserAgentMiddlewareTest(TestCase):
         self.assertEqual(req.headers['User-Agent'], b'default_useragent')
 
     def test_remove_agent(self):
-        # settings UESR_AGENT to None should remove the user agent
+        # settings USER_AGENT to None should remove the user agent
         spider, mw = self.get_spider_and_mw('default_useragent')
         spider.user_agent = None
         mw.spider_opened(spider)

From 41734bb5c1310760db012ff705ce98910b955791 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Sun, 18 Dec 2022 21:46:49 -0500
Subject: [PATCH 34/58] spelling: unencodeable

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index a1e3e25e0..07264827b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -4700,7 +4700,7 @@ Scrapy 0.22.1 (released 2014-02-08)
 - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
 - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
 - Fix tests for Travis-CI build (:commit:`76c7e20`)
-- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
+- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
 - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
 - Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
 - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)

From 0a84ce448cdbbd077ec9e39a892b0c14fdb77217 Mon Sep 17 00:00:00 2001
From: Matt Mayfield <mattmayf@umich.edu>
Date: Mon, 19 Dec 2022 18:09:43 -0500
Subject: [PATCH 35/58] Fix InstrumentedFeedSlot

I accidentally called the wrong super method in overriden
finish_exporting
---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 0d5b0f08e..c66ce804b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -690,7 +690,7 @@ class InstrumentedFeedSlot(_FeedSlot):
 
     def finish_exporting(self):
         self.update_listener('finish')
-        super().start_exporting()
+        super().finish_exporting()
 
     @classmethod
     def subscribe__listener(cls, listener):

From 517ed0749b81b234dd99f3c2538dc64c7513a80f Mon Sep 17 00:00:00 2001
From: Malkiz223 <malkiz@yandex.ru>
Date: Sat, 31 Dec 2022 00:51:43 +0300
Subject: [PATCH 36/58] Fix overwriting repeated headers

---
 scrapy/core/http2/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index e197790f7..ff82ad0fd 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -332,7 +332,7 @@ class Stream:
 
     def receive_headers(self, headers: List[HeaderTuple]) -> None:
         for name, value in headers:
-            self._response['headers'][name] = value
+            self._response['headers'].appendlist(name, value)
 
         # Check if we exceed the allowed max data size which can be received
         expected_size = int(self._response['headers'].get(b'Content-Length', -1))

From deaf1fb6cf513db8119acfe70fd3e08f761f7ede Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Sat, 31 Dec 2022 16:11:45 -0300
Subject: [PATCH 37/58] Stop using setup.py (#5776)

---
 .github/workflows/publish.yml | 4 ++--
 tox.ini                       | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 991b0b6e8..eee9a4f02 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -24,8 +24,8 @@ jobs:
     - name: Publish to PyPI
       if: steps.check-release-tag.outputs.release_tag == 'true'
       run: |
-        pip install --upgrade setuptools wheel twine
-        python setup.py sdist bdist_wheel
+        pip install --upgrade build twine
+        python -m build
         export TWINE_USERNAME=__token__
         export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
         twine upload dist/*
diff --git a/tox.ini b/tox.ini
index 4d0f0291b..0e156d63b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -75,8 +75,9 @@ commands =
 basepython = python3
 deps =
     twine==4.0.1
+    build==0.9.0
 commands =
-    python setup.py sdist
+    python -m build --sdist
     twine check dist/*
 
 [pinned]

From c4d7f5e7a996e7e5e5c32e947c026d29df5315d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 2 Jan 2023 13:41:08 +0100
Subject: [PATCH 38/58] Use CLang to build Reppy

---
 .github/workflows/checks.yml       | 12 ++++++++++--
 .github/workflows/tests-ubuntu.yml | 13 ++++++++++---
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 8c1ae4bd3..e5b0306b3 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -14,10 +14,12 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: flake8
-        # Pylint requires installing reppy, which does not support Python 3.9
-        # https://github.com/seomoz/reppy/issues/122
+        # Pylint requires installing reppy, which:
+        # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122
+        # - Requires CLang to build https://github.com/seomoz/reppy/issues/132
         - python-version: 3.8
           env:
+            CC: clang
             TOXENV: pylint
         - python-version: 3.7
           env:
@@ -37,6 +39,12 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Set up Clang
+      uses: egor-tensin/setup-clang@v1
+      with:
+        version: latest
+        platform: x64
+
     - name: Run check
       env: ${{ matrix.env }}
       run: |
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 9c3ce8115..4ad722ad5 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -38,11 +38,12 @@ jobs:
           env:
             TOXENV: pypy3-pinned
 
-        # extras
-        # extra-deps includes reppy, which does not support Python 3.9
-        # https://github.com/seomoz/reppy/issues/122
+        # extra-deps includes reppy, which:
+        # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122
+        # - Requires CLang to build https://github.com/seomoz/reppy/issues/132
         - python-version: 3.8
           env:
+            CC: clang
             TOXENV: extra-deps
 
     steps:
@@ -59,6 +60,12 @@ jobs:
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev
 
+    - name: Set up Clang
+      uses: egor-tensin/setup-clang@v1
+      with:
+        version: latest
+        platform: x64
+
     - name: Run tests
       env: ${{ matrix.env }}
       run: |

From e47ada2c7cc7c31f31e54652d5b7f893678198af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 2 Jan 2023 13:49:25 +0100
Subject: [PATCH 39/58] Add CC to tox.ini:[testenv]passenv

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 0e156d63b..865ef5383 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,6 +21,7 @@ deps =
     # Extras
     botocore>=1.4.87
 passenv =
+    CC
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY

From 6084e8f6270acd0a8e7a0081ab0b60dfbdf56933 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 2 Jan 2023 14:00:37 +0100
Subject: [PATCH 40/58] Do not use -U for pip in Tox

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 0e156d63b..bf646242b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,7 +32,7 @@ download = true
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =
-    pip install -U -ctests/upper-constraints.txt {opts} {packages}
+    python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
 
 [testenv:typing]
 basepython = python3
@@ -107,7 +107,7 @@ deps =
 setenv =
     _SCRAPY_PINNED=true
 install_command =
-    pip install -U {opts} {packages}
+    python -I -m pip install {opts} {packages}
 
 [testenv:pinned]
 deps =

From 016d1de64eebf6abd929ab7fb0037334da66677b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 2 Jan 2023 14:13:04 +0100
Subject: [PATCH 41/58] Remove Reppy from CI

---
 .github/workflows/checks.yml       | 12 +-----------
 .github/workflows/tests-ubuntu.yml | 12 +-----------
 tox.ini                            |  8 ++------
 3 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index e5b0306b3..e9f9a6aea 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -14,12 +14,8 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: flake8
-        # Pylint requires installing reppy, which:
-        # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122
-        # - Requires CLang to build https://github.com/seomoz/reppy/issues/132
-        - python-version: 3.8
+        - python-version: "3.11"
           env:
-            CC: clang
             TOXENV: pylint
         - python-version: 3.7
           env:
@@ -39,12 +35,6 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Set up Clang
-      uses: egor-tensin/setup-clang@v1
-      with:
-        version: latest
-        platform: x64
-
     - name: Run check
       env: ${{ matrix.env }}
       run: |
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 4ad722ad5..8fcf90a18 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -38,12 +38,8 @@ jobs:
           env:
             TOXENV: pypy3-pinned
 
-        # extra-deps includes reppy, which:
-        # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122
-        # - Requires CLang to build https://github.com/seomoz/reppy/issues/132
-        - python-version: 3.8
+        - python-version: "3.11"
           env:
-            CC: clang
             TOXENV: extra-deps
 
     steps:
@@ -60,12 +56,6 @@ jobs:
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev
 
-    - name: Set up Clang
-      uses: egor-tensin/setup-clang@v1
-      with:
-        version: latest
-        platform: x64
-
     - name: Run tests
       env: ${{ matrix.env }}
       run: |
diff --git a/tox.ini b/tox.ini
index 865ef5383..cc529fc70 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,6 @@ deps =
     # Extras
     botocore>=1.4.87
 passenv =
-    CC
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
@@ -64,8 +63,7 @@ commands =
     flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
-# reppy does not support Python 3.9+
-basepython = python3.8
+basepython = python3
 deps =
     {[testenv:extra-deps]deps}
     pylint==2.15.6
@@ -128,8 +126,7 @@ setenv =
     {[pinned]setenv}
 
 [testenv:extra-deps]
-# reppy does not support Python 3.9+
-basepython = python3.8
+basepython = python3
 deps =
     {[testenv]deps}
     boto
@@ -137,7 +134,6 @@ deps =
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
-    reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
     Twisted[http2]>=17.9.0

From fd6742e8117eb76f4a717b33b24790c1419bd838 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 2 Jan 2023 19:13:31 +0100
Subject: [PATCH 42/58] Fix tests for pygments 2.14 (#5783)

---
 tests/test_utils_display.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 9ec8311d9..1d1fb50e6 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -7,17 +7,27 @@ from scrapy.utils.display import pformat, pprint
 
 class TestDisplay(TestCase):
     object = {'a': 1}
-    colorized_string = (
-        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
-        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
-    )
+    colorized_strings = {
+        (
+            (
+                "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+                "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
+            )
+            + suffix
+        )
+        for suffix in (
+            # https://github.com/pygments/pygments/issues/2313
+            "\n",  # pygments ≤ 2.13
+            "\x1b[37m\x1b[39;49;00m\n",  # pygments ≥ 2.14
+        )
+    }
     plain_string = "{'a': 1}"
 
     @mock.patch('sys.platform', 'linux')
     @mock.patch("sys.stdout.isatty")
     def test_pformat(self, isatty):
         isatty.return_value = True
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch("sys.stdout.isatty")
     def test_pformat_dont_colorize(self, isatty):
@@ -33,7 +43,7 @@ class TestDisplay(TestCase):
     def test_pformat_old_windows(self, isatty, version):
         isatty.return_value = True
         version.return_value = '10.0.14392'
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch('sys.platform', 'win32')
     @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
@@ -53,7 +63,7 @@ class TestDisplay(TestCase):
         isatty.return_value = True
         version.return_value = '10.0.14393'
         terminal_processing.return_value = True
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch('sys.platform', 'linux')
     @mock.patch("sys.stdout.isatty")

From 724b0332878b5df502dc9504aec69c47fb1ea0bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Jan 2023 19:51:07 +0100
Subject: [PATCH 43/58] Fix CI issues related to asyncio (#5782)

---
 scrapy/utils/defer.py       |  7 +++----
 scrapy/utils/reactor.py     | 28 ++++++++++++++++++++++++----
 tests/test_utils_asyncio.py |  3 +++
 3 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index ddacfaa49..0da2e3638 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -26,7 +26,7 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
+from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -267,7 +267,7 @@ def deferred_from_coro(o) -> Any:
             # that use asyncio, e.g. "await asyncio.sleep(1)"
             return ensureDeferred(o)
         # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-        event_loop = get_asyncio_event_loop_policy().get_event_loop()
+        event_loop = _get_asyncio_event_loop()
         return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
     return o
 
@@ -318,8 +318,7 @@ def deferred_to_future(d: Deferred) -> Future:
                 d = treq.get('https://example.com/additional')
                 additional_response = await deferred_to_future(d)
     """
-    policy = get_asyncio_event_loop_policy()
-    return d.asFuture(policy.get_event_loop())
+    return d.asFuture(_get_asyncio_event_loop())
 
 
 def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index e6b8de292..6a051d8b0 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,6 +1,7 @@
 import asyncio
 import sys
 from contextlib import suppress
+from warnings import catch_warnings, filterwarnings
 
 from twisted.internet import asyncioreactor, error
 
@@ -81,6 +82,10 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
+def _get_asyncio_event_loop():
+    return set_asyncio_event_loop(None)
+
+
 def set_asyncio_event_loop(event_loop_path):
     """Sets and returns the event loop with specified import path."""
     policy = get_asyncio_event_loop_policy()
@@ -90,11 +95,26 @@ def set_asyncio_event_loop(event_loop_path):
         asyncio.set_event_loop(event_loop)
     else:
         try:
-            event_loop = policy.get_event_loop()
+            with catch_warnings():
+                # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning
+                # is emitted about the lack of a current event loop, because in
+                # Python 3.14 and later `get_event_loop` will raise a
+                # RuntimeError in that event. Because our code is already
+                # prepared for that future behavior, we ignore the deprecation
+                # warning.
+                filterwarnings(
+                    "ignore",
+                    message="There is no current event loop",
+                    category=DeprecationWarning,
+                )
+                event_loop = policy.get_event_loop()
         except RuntimeError:
-            # `get_event_loop` is expected to fail when called from a new thread
-            # with no asyncio event loop yet installed. Such is the case when
-            # called from `scrapy shell`
+            # `get_event_loop` raises RuntimeError when called with no asyncio
+            # event loop yet installed in the following scenarios:
+            # - From a thread other than the main thread. For example, when
+            #   using ``scrapy shell``.
+            # - Previsibly on Python 3.14 and later.
+            #   https://github.com/python/cpython/issues/100160#issuecomment-1345581902
             event_loop = policy.new_event_loop()
             asyncio.set_event_loop(event_loop)
     return event_loop
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 741c6a505..42780ace7 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -14,6 +14,9 @@ class AsyncioTest(TestCase):
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
     def test_install_asyncio_reactor(self):
+        from twisted.internet import reactor as original_reactor
         with warnings.catch_warnings(record=True) as w:
             install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
             self.assertEqual(len(w), 0)
+        from twisted.internet import reactor
+        assert original_reactor == reactor

From 16546564f673b0d58c55584870149424abc73fee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Jan 2023 20:16:54 +0100
Subject: [PATCH 44/58] =?UTF-8?q?Minimum=20cryptography:=203.3=20=E2=86=92?=
 =?UTF-8?q?=203.4.6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 setup.py | 2 +-
 tox.ini  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 27445cad0..bdae28047 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     'Twisted>=18.9.0',
-    'cryptography>=3.3',
+    'cryptography>=3.4.6',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
diff --git a/tox.ini b/tox.ini
index 84e83ac2a..520d90303 100644
--- a/tox.ini
+++ b/tox.ini
@@ -81,7 +81,7 @@ commands =
 
 [pinned]
 deps =
-    cryptography==3.3
+    cryptography==3.4.6
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0

From 1ab900659e32b6792aabfb30a51ce117d2578cfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 11 Jan 2023 14:05:13 +0100
Subject: [PATCH 45/58] =?UTF-8?q?Fix=20typo:=20finis=20=E2=86=92=20finish?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c66ce804b..feaba5dab 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -700,7 +700,7 @@ class InstrumentedFeedSlot(_FeedSlot):
 class IsExportingListener:
     """When subscribed to InstrumentedFeedSlot, keeps track of when
     a call to start_exporting has been made without a closing call to
-    finish_exporting and when a call to finis_exporting has been made
+    finish_exporting and when a call to finish_exporting has been made
     before a call to start_exporting."""
     def __init__(self):
         self.start_without_finish = False

From d9f6de6bf5d3b59e4230b7920a079db5c8cc5ffa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 11 Jan 2023 15:59:45 +0100
Subject: [PATCH 46/58] Add a test for receiving duplicate headers

---
 tests/test_downloader_handlers.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 3dc2745a0..b1e711aff 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -214,6 +214,12 @@ class LargeChunkedFileResource(resource.Resource):
         return server.NOT_DONE_YET
 
 
+class DuplicateHeaderResource(resource.Resource):
+    def render(self, request):
+        request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
+        return b""
+
+
 class HttpTestCase(unittest.TestCase):
     scheme = 'http'
     download_handler_cls: Type = HTTPDownloadHandler
@@ -239,6 +245,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"contentlength", ContentLengthHeaderResource())
         r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
         r.putChild(b"largechunkedfile", LargeChunkedFileResource())
+        r.putChild(b"duplicate-header", DuplicateHeaderResource())
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
@@ -407,6 +414,16 @@ class HttpTestCase(unittest.TestCase):
             HtmlResponse,
         )
 
+    def test_get_duplicate_header(self):
+        def _test(response):
+            self.assertEqual(
+                response.headers.getlist(b'Set-Cookie'),
+                [b'a=b', b'c=d'],
+            )
+
+        request = Request(self.getURL('duplicate-header'))
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""

From b0ec38876d52febb6811348961b75247ab4fe1c8 Mon Sep 17 00:00:00 2001
From: spav <silvio.pavanetto@jobtome.com>
Date: Wed, 11 Jan 2023 19:32:33 +0100
Subject: [PATCH 47/58] Fix docstring for extract_links specifying duplicates
 case

---
 scrapy/linkextractors/lxmlhtml.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 14eb24862..8a4175d49 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -195,7 +195,8 @@ class LxmlLinkExtractor:
         Only links that match the settings passed to the ``__init__`` method of
         the link extractor are returned.
 
-        Duplicate links are omitted.
+        Duplicate links are omitted if the ``unique`` parameter is set to ``True``, 
+        otherwise they are returned.
         """
         base_url = get_base_url(response)
         if self.restrict_xpaths:

From b386c648643529f3f2d9e4d1783632840e6fe270 Mon Sep 17 00:00:00 2001
From: spav <silvio.pavanetto@jobtome.com>
Date: Wed, 11 Jan 2023 19:58:42 +0100
Subject: [PATCH 48/58] Fix tests for allowing duplicates in extract_links

---
 tests/sample_data/link_extractor/linkextractor.html | 1 +
 tests/test_http_response.py                         | 1 +
 tests/test_linkextractors.py                        | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index e3a2a4145..29075602d 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -13,6 +13,7 @@
       </div>
       <a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
       <a href='sample3.html'>sample 3 repetition</a>
+      <a href='sample3.html'>sample 3 repetition</a>
       <a href='sample3.html#foo'>sample 3 repetition with fragment</a>
       <a href='http://www.google.com/something'></a>
       <a href='http://example.com/innertag.html'><strong>inner</strong> tag</a>
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 74e170ec0..07aef2ee1 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -541,6 +541,7 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com/sample2.html',
             'http://example.com/sample3.html',
             'http://example.com/sample3.html',
+            'http://example.com/sample3.html',
             'http://example.com/sample3.html#foo',
             'http://www.google.com/something',
             'http://example.com/innertag.html'
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index e28dc9bdb..6c34a96a0 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -53,6 +53,7 @@ class Base:
                 Link(url='http://example.com/sample2.html', text='sample 2'),
                 Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
@@ -64,6 +65,7 @@ class Base:
                 Link(url='http://example.com/sample2.html', text='sample 2'),
                 Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
             ])
 

From faa5bd0f6b688eaef18a6245f2d703e8fc8ff684 Mon Sep 17 00:00:00 2001
From: silviopavanetto <silviopavanetto@gmail.com>
Date: Wed, 11 Jan 2023 20:30:57 +0100
Subject: [PATCH 49/58] Update scrapy/linkextractors/lxmlhtml.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/linkextractors/lxmlhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8a4175d49..f772df987 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -195,7 +195,7 @@ class LxmlLinkExtractor:
         Only links that match the settings passed to the ``__init__`` method of
         the link extractor are returned.
 
-        Duplicate links are omitted if the ``unique`` parameter is set to ``True``, 
+        Duplicate links are omitted if the ``unique`` attribute is set to ``True``, 
         otherwise they are returned.
         """
         base_url = get_base_url(response)

From 36b89a4b20f12a84123930664eab888dfdf3dd8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 12 Jan 2023 08:36:31 +0100
Subject: [PATCH 50/58] Remove trailing whitespace

---
 scrapy/linkextractors/lxmlhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index f772df987..3f90ed84a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -195,7 +195,7 @@ class LxmlLinkExtractor:
         Only links that match the settings passed to the ``__init__`` method of
         the link extractor are returned.
 
-        Duplicate links are omitted if the ``unique`` attribute is set to ``True``, 
+        Duplicate links are omitted if the ``unique`` attribute is set to ``True``,
         otherwise they are returned.
         """
         base_url = get_base_url(response)

From 93ad6a4bc2fd2d453a37961b817a8a2a85f589c4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Jan 2023 20:39:44 +0400
Subject: [PATCH 51/58] Simplify code for modern pyOpenSSL.

---
 scrapy/core/downloader/tls.py |  8 ++++----
 scrapy/utils/ssl.py           | 14 ++------------
 tests/mockserver.py           |  3 +--
 3 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 7d67a426f..65028d21f 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -17,10 +17,10 @@ METHOD_TLSv12 = 'TLSv1.2'
 
 
 openssl_methods = {
-    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
-    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
-    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
+    METHOD_TLS: SSL.SSLv23_METHOD,      # protocol negotiation (recommended)
+    METHOD_TLSv10: SSL.TLSv1_METHOD,    # TLS 1.0 only
+    METHOD_TLSv11: SSL.TLSv1_1_METHOD,  # TLS 1.1 only
+    METHOD_TLSv12: SSL.TLSv1_2_METHOD,  # TLS 1.2 only
 }
 
 
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index ea4dde882..98efd91c7 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,14 +1,9 @@
-import OpenSSL
+import OpenSSL.SSL
 import OpenSSL._util as pyOpenSSLutil
 
 from scrapy.utils.python import to_unicode
 
 
-# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
-# Using the binding directly, as this code does, requires cryptography 2.4.
-SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
-
-
 def ffi_buf_to_string(buf):
     return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
@@ -22,9 +17,6 @@ def x509name_to_string(x509name):
 
 
 def get_temp_key_info(ssl_object):
-    if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'):  # requires OpenSSL 1.0.2
-        return None
-
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
     temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
     if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
@@ -55,7 +47,5 @@ def get_temp_key_info(ssl_object):
 
 
 def get_openssl_version():
-    system_openssl = OpenSSL.SSL.SSLeay_version(
-        OpenSSL.SSL.SSLEAY_VERSION
-    ).decode('ascii', errors='replace')
+    system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
     return f'{OpenSSL.version.__version__} ({system_openssl})'
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 6d2d95692..4fd3adce7 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -19,7 +19,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
 from scrapy.utils.test import get_testenv
 
 
@@ -350,7 +349,7 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
     if cipher_string:
         ctx = factory.getContext()
         # disabling TLS1.3 because it unconditionally enables some strong ciphers
-        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3)
+        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3)
         ctx.set_cipher_list(to_bytes(cipher_string))
     return factory
 

From fb52918d23b37cd2581aa39ebbd422b017ec8051 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Jan 2023 20:46:55 +0400
Subject: [PATCH 52/58] Set OP_LEGACY_SERVER_CONNECT to support some old
 servers when using OpenSSL 3.

---
 scrapy/core/downloader/contextfactory.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 4abde2238..bc6ad34d8 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -61,7 +61,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
     def getContext(self, hostname=None, port=None):
-        return self.getCertificateOptions().getContext()
+        ctx = self.getCertificateOptions().getContext()
+        ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
+        return ctx
 
     def creatorForNetloc(self, hostname, port):
         return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),

From 43ab8bd16acdabc8be64b421b030fb6e1751e47b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 13 Jan 2023 20:51:07 +0400
Subject: [PATCH 53/58] Roll back the get_openssl_version() type change.

---
 scrapy/utils/ssl.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 98efd91c7..ce211bf9b 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -47,5 +47,7 @@ def get_temp_key_info(ssl_object):
 
 
 def get_openssl_version():
-    system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
+    system_openssl = OpenSSL.SSL.SSLeay_version(
+        OpenSSL.SSL.SSLEAY_VERSION
+    ).decode('ascii', errors='replace')
     return f'{OpenSSL.version.__version__} ({system_openssl})'

From caaeb235a08a250a9438c97259f2a74ae2ca8bbd Mon Sep 17 00:00:00 2001
From: Serhii A <aserhii@protonmail.com>
Date: Tue, 17 Jan 2023 13:52:41 +0200
Subject: [PATCH 54/58] =?UTF-8?q?scrapy.utils.console.DEFAULT=5FPYTHON=5FS?=
 =?UTF-8?q?HELLS:=20OrderedDict=20=E2=86=92=20dict=20(#5795)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/utils/console.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 1bc0bd45f..4828c7767 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,5 +1,4 @@
 from functools import wraps
-from collections import OrderedDict
 
 
 def _embed_ipython_shell(namespace={}, banner=''):
@@ -63,12 +62,12 @@ def _embed_standard_shell(namespace={}, banner=''):
     return wrapper
 
 
-DEFAULT_PYTHON_SHELLS = OrderedDict([
-    ('ptpython', _embed_ptpython_shell),
-    ('ipython', _embed_ipython_shell),
-    ('bpython', _embed_bpython_shell),
-    ('python', _embed_standard_shell),
-])
+DEFAULT_PYTHON_SHELLS = {
+    'ptpython': _embed_ptpython_shell,
+    'ipython': _embed_ipython_shell,
+    'bpython': _embed_bpython_shell,
+    'python': _embed_standard_shell,
+}
 
 
 def get_shell_embed_func(shells=None, known_shells=None):

From f449ee53778b038f9ecb1feccb2fa9baa40e1f56 Mon Sep 17 00:00:00 2001
From: Tobias Mayr <tobistian@gmail.com>
Date: Thu, 19 Jan 2023 18:44:55 +0000
Subject: [PATCH 55/58] Fix SMTP STARTTLS for Twisted >= 21.2.0 (#5406)

---
 scrapy/mail.py     | 28 +++++++++++++++++++++-------
 tests/test_mail.py | 16 ++++++++++++++++
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 2a25ccd44..b8cc28335 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -12,7 +12,9 @@ from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
 
+from twisted.python.versions import Version
 from twisted.internet import defer, ssl
+from twisted import version as twisted_version
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
@@ -126,16 +128,11 @@ class MailSender:
                       'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
-        # Import twisted.mail here because it is not available in python3
         from twisted.internet import reactor
-        from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(
-            self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
-            heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
-        )
-        factory.noisy = False
+
+        factory = self._create_sender_factory(to_addrs, msg, d)
 
         if self.smtpssl:
             reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
@@ -143,3 +140,20 @@ class MailSender:
             reactor.connectTCP(self.smtphost, self.smtpport, factory)
 
         return d
+
+    def _create_sender_factory(self, to_addrs, msg, d):
+        from twisted.mail.smtp import ESMTPSenderFactory
+
+        factory_keywords = {
+            'heloFallback': True,
+            'requireAuthentication': False,
+            'requireTransportSecurity': self.smtptls
+        }
+
+        # Newer versions of twisted require the hostname to use STARTTLS
+        if twisted_version >= Version('twisted', 21, 2, 0):
+            factory_keywords['hostname'] = self.smtphost
+
+        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords)
+        factory.noisy = False
+        return factory
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 9b248fbfa..fd02020ee 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -4,6 +4,11 @@ import unittest
 from io import BytesIO
 from email.charset import Charset
 
+from twisted.internet._sslverify import ClientTLSOptions
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.versions import Version
+from twisted.internet import defer
+from twisted import version as twisted_version
 from scrapy.mail import MailSender
 
 
@@ -121,6 +126,17 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(text.get_charset(), Charset('utf-8'))
         self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
 
+    def test_create_sender_factory_with_host(self):
+        mailsender = MailSender(debug=False, smtphost='smtp.testhost.com')
+
+        factory = mailsender._create_sender_factory(to_addrs=['test@scrapy.org'], msg='test', d=defer.Deferred())
+
+        context = factory.buildProtocol('test@scrapy.org').context
+        if twisted_version >= Version('twisted', 21, 2, 0):
+            self.assertIsInstance(context, ClientTLSOptions)
+        else:
+            self.assertIsInstance(context, ClientContextFactory)
+
 
 if __name__ == "__main__":
     unittest.main()

From 8270df754d5caa7e8115432923197f09b4ebc78f Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 20 Jan 2023 07:55:16 -0300
Subject: [PATCH 56/58] Set `FEED_EXPORT_ENCODING='utf-8'` in the default
 template

---
 docs/topics/feed-exports.rst                     | 5 +++++
 scrapy/templates/project/module/settings.py.tmpl | 1 +
 2 files changed, 6 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index a620e2c04..7b662f34d 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -515,6 +515,11 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
 
 Use ``utf-8`` if you want UTF-8 for JSON too.
 
+.. versionchanged:: 2.8
+   The :command:`startproject` command now sets this setting to
+   ``utf-8`` in the generated
+   ``settings.py`` file.
+
 .. setting:: FEED_EXPORT_FIELDS
 
 FEED_EXPORT_FIELDS
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index bbf60982c..2f6df5abc 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -90,3 +90,4 @@ ROBOTSTXT_OBEY = True
 # Set settings whose default value is deprecated to a future-proof value
 REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
 TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+FEED_EXPORT_ENCODING = 'utf-8'

From 973f0cf5678adcec36aaf5d0ceb860198f34de4a Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 20 Jan 2023 08:23:05 -0300
Subject: [PATCH 57/58] fix: line break

---
 docs/topics/feed-exports.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7b662f34d..8775a99d0 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -517,8 +517,7 @@ Use ``utf-8`` if you want UTF-8 for JSON too.
 
 .. versionchanged:: 2.8
    The :command:`startproject` command now sets this setting to
-   ``utf-8`` in the generated
-   ``settings.py`` file.
+   ``utf-8`` in the generated ``settings.py`` file.
 
 .. setting:: FEED_EXPORT_FIELDS
 

From b6118480299384c88d194bf09ad60b67a36395ed Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 20 Jan 2023 08:33:35 -0300
Subject: [PATCH 58/58] fix(docs): Change `versionchanged` value

---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 8775a99d0..8f96b1154 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -515,7 +515,7 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
 
 Use ``utf-8`` if you want UTF-8 for JSON too.
 
-.. versionchanged:: 2.8
+.. versionchanged:: VERSION
    The :command:`startproject` command now sets this setting to
    ``utf-8`` in the generated ``settings.py`` file.